diff --git a/scripts/APIv1-frontend.js b/scripts/APIv1-frontend.js new file mode 100644 index 00000000000..a16035260ea --- /dev/null +++ b/scripts/APIv1-frontend.js @@ -0,0 +1,299 @@ +#!/usr/bin/env node + +const fs = require('fs').promises; +const path = require('path'); +const {globSync} = require('glob'); +const core = require('@actions/core'); + +const version = 'v1-frontend'; + +// Define the path to the entries and the API output directory +const entriesGlob = 'entries/*/*.json'; +const apiDirectory = 'api/frontend/v1'; + +// URL to fetch categories data from +const categoriesUrl = 'https://raw.githubusercontent.com/2factorauth/2fa.directory/refs/heads/master/data/categories.json'; + +/** + * Read and parse a JSON file asynchronously. + * + * @param {string} filePath - The path to the JSON file. + * @returns {Promise} - The parsed JSON object. + */ +const readJSONFile = (filePath) => fs.readFile(filePath, 'utf8'). + then(JSON.parse); + +/** + * Write a JSON object to a file asynchronously, ensuring the directory exists. + * + * @param {string} filePath - The path to the output file. + * @param {Object} data - The JSON object to write. + * @returns {Promise} + */ +const writeJSONFile = async (filePath, data) => { + const dir = path.dirname(filePath); + await fs.mkdir(dir, {recursive: true}); + await fs.writeFile( + filePath, + JSON.stringify(data, null, process.env.NODE_ENV !== 'production' ? 2:0), + ); +}; + +/** + * Fetch and parse JSON data from a URL. + * + * @param {string} url - The URL to fetch data from. + * @returns {Promise} - The parsed JSON object. + */ +const fetchJSONFromUrl = async (url) => { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Failed to fetch JSON from ${url}: ${response.statusText}`); + } + return response.json(); +}; + +/** + * Process entries by loading and transforming them. + * + * @param {string[]} files - Array of file paths to process. + * @returns {Promise} - An object containing all processed entries. + */ +const processEntries = async (files) => { + const entries = {}; + + await Promise.all( + files.map(async (file) => { + const data = await readJSONFile(file); + const [mainDomain, entry] = Object.entries(data)[0]; + + // Add the main domain entry + entries[mainDomain] = entry; + }), + ); + + return entries; +}; + +/** + * Generate the API files from the processed entries. + * + * @param {Object} entries - The processed entries. + * @param {Object} categoriesData - The categories data fetched from the URL. + * @returns {Promise} + */ +const generateApi = async (entries, categoriesData) => { + const categoriesByRegion = {}; + const entryCountsByRegion = {}; + const categoriesUsedByRegion = {}; + + // Initialize 'int' (international) region + categoriesByRegion.int = {}; + categoriesUsedByRegion.int = new Set(); + + // Collect all regions from entries that have regions arrays + const allRegions = new Set(); + for (const entry of Object.values(entries)) { + if (entry.regions) { + for (const region of entry.regions) { + const regionName = region.replace('-', ''); + allRegions.add(regionName); + } + } + } + + // Process each entry + await Promise.all( + Object.entries(entries). + map(([domain, entry]) => processEntry(domain, entry)), + ); + + // Write 'int' region files + await writeRegionFiles('int'); + + // Write other regions + for (const region of Object.keys(categoriesByRegion)) { + if (region !== 'int') { + await writeRegionFiles(region); + } + } + + /** + * Process a single entry and add it to the appropriate regions. + * + * @param {string} domain - The domain of the entry. + * @param {Object} entry - The entry data. + */ + async function processEntry(domain, entry) { + const apiEntry = { + methods: entry.tfa, + domain: entry.domain, + 'custom-software': entry['custom-software'], + 'custom-hardware': entry['custom-hardware'], + contact: entry.contact, + notes: entry.notes, + img: entry.img, + documentation: entry.documentation, + recovery: entry.recovery, + }; + + // Always include in 'int' region + addEntryToRegion('int', entry.categories, domain, apiEntry); + + // Determine which regions the entry should be included in + const {includeRegions, explicitlyIncludedRegions} = getIncludeRegions( + entry); + + // For each region, add the entry to the region's categories + for (const region of includeRegions) { + if (region === 'int') continue; // already processed + addEntryToRegion(region, entry.categories, domain, apiEntry); + + // If the entry explicitly includes the region, increment the entry count + if (explicitlyIncludedRegions.has(region)) { + incrementEntryCount(region); + } + } + } + + /** + * Determine which regions an entry should be included in. + * + * @param {Object} entry - The entry data. + * @returns {Object} - An object containing includeRegions and explicitlyIncludedRegions sets. + */ + function getIncludeRegions(entry) { + const includeRegions = new Set(allRegions); + const excludeRegions = new Set(); + const explicitlyIncludedRegions = new Set(); + let hasExplicitInclude = false; + + includeRegions.delete('int'); // Exclude 'int' from processing here + + if (entry.regions && entry.regions.length > 0) { + for (const region of entry.regions) { + const regionName = region.replace('-', ''); + if (region.startsWith('-')) { + excludeRegions.add(regionName); + } else { + explicitlyIncludedRegions.add(regionName); + hasExplicitInclude = true; + } + } + + if (hasExplicitInclude) { + // If there are explicit includes, set includeRegions to only those + includeRegions.clear(); + for (const region of explicitlyIncludedRegions) { + includeRegions.add(region); + } + } + + // Exclude regions from includeRegions + for (const region of excludeRegions) { + includeRegions.delete(region); + } + } + + return { + includeRegions, + explicitlyIncludedRegions, + }; + } + + /** + * Add an entry to the specified region and categories. + * + * @param {string} region - The region to add the entry to. + * @param {string[]} categories - The categories of the entry. + * @param {string} domain - The domain of the entry. + * @param {Object} apiEntry - The entry data to add. + */ + function addEntryToRegion(region, categories, domain, apiEntry) { + if (!categoriesByRegion[region]) { + categoriesByRegion[region] = {}; + categoriesUsedByRegion[region] = new Set(); + entryCountsByRegion[region] = 0; + } + + categories.forEach((category) => { + categoriesByRegion[region][category] = categoriesByRegion[region][category] || + {}; + categoriesByRegion[region][category][domain] = apiEntry; + categoriesUsedByRegion[region].add(category); + }); + } + + /** + * Increment the entry count for a region. + * + * @param {string} region - The region to increment the count for. + */ + function incrementEntryCount(region) { + if (!entryCountsByRegion[region]) { + entryCountsByRegion[region] = 0; + } + entryCountsByRegion[region] += 1; + } + + /** + * Write category files for a region if it meets the entry count threshold. + * + * @param {string} region - The region to write files for. + */ + async function writeRegionFiles(region) { + const entryCount = entryCountsByRegion[region] || 0; + + if (region !== 'int' && entryCount < 10) { + core.info(`Ignoring '${region}' as it only has ${entryCount} entrie(s).`); + return; + } + + const regionDir = path.join(apiDirectory, region); + + // Write category files + const categoryWrites = Object.entries(categoriesByRegion[region]). + sort(). + map(([category, entries]) => + writeJSONFile(path.join(regionDir, `${category}.json`), entries), + ); + + // Write categories.json file + const categoriesUsed = categoriesUsedByRegion[region]; + const categoriesDataForRegion = Object.fromEntries( + [...categoriesUsed].filter((category) => categoriesData[category]). + sort(). + map((category) => [category, categoriesData[category]]), + ); + + const categoriesWrite = writeJSONFile( + path.join(regionDir, 'categories.json'), + categoriesDataForRegion, + ); + + await Promise.all([...categoryWrites, categoriesWrite]); + } +}; + +/** + * Main function to orchestrate the loading, processing, and API generation. + */ +(async () => { + try { + core.info(`Generating API ${version}`); + + // Fetch categories data from URL + const categoriesData = await fetchJSONFromUrl(categoriesUrl); + + // Get all JSON entry files + const files = globSync(entriesGlob); + + // Process entries and generate the API + const entries = await processEntries(files); + await generateApi(entries, categoriesData); + + core.info(`API ${version} generation completed successfully`); + } catch (error) { + core.setFailed(error.message); + } +})();