From 92480d1474966fd61c1c47027afebdb28fd37caa Mon Sep 17 00:00:00 2001 From: linhcandoit Date: Fri, 2 Aug 2024 09:49:05 +0700 Subject: [PATCH] projects now empty --- .../src/controllers/crawl.controller.ts | 7 ++- apps/server/src/crawl.ts | 4 +- apps/server/src/crawl/classificationCase.ts | 20 ++++++-- apps/server/src/crawl/handleCase.ts | 39 ++++++++++++++- .../utils/filterNewFeaturesAndGlossaries.ts | 31 ++++++++++++ .../src/crawl/utils/filterNewProject.ts | 30 ------------ .../src/crawl/utils/filterNewSocials.ts | 17 +++++++ apps/server/src/crawl/utils/filterProject.ts | 47 +++++++++++++++++++ apps/server/src/crawl/utils/removeTrash.ts | 11 +++++ apps/server/src/crawl/utils/updateFeatures.ts | 7 +++ apps/server/src/test.ts | 17 ------- apps/server/src/utils/fs/fsWrapper.ts | 11 +++++ apps/server/src/utils/getFilePathDepth.ts | 4 ++ projects/DeFi/DEX/Coinlink/detail.md | 0 projects/DeFi/DEX/Coinlink/features.json | 6 --- projects/DeFi/DEX/Coinlink/info.json | 37 --------------- projects/DeFi/DEX/Coinlink/logo.png | 0 projects/DeFi/DEX/Geninus/detail.md | 0 projects/DeFi/DEX/Geninus/info.json | 34 -------------- projects/DeFi/DEX/Geninus/logo.png | 0 projects/DeFi/category.json | 28 ----------- projects/DeFi/features.json | 12 ----- projects/DeFi/glossaries.json | 10 ---- projects/Wallets/category.json | 8 ---- projects/projects.json | 25 ++-------- projects/socials.json | 27 ----------- 26 files changed, 189 insertions(+), 243 deletions(-) create mode 100644 apps/server/src/crawl/utils/filterNewFeaturesAndGlossaries.ts delete mode 100644 apps/server/src/crawl/utils/filterNewProject.ts create mode 100644 apps/server/src/crawl/utils/filterNewSocials.ts create mode 100644 apps/server/src/crawl/utils/filterProject.ts create mode 100644 apps/server/src/crawl/utils/removeTrash.ts create mode 100644 apps/server/src/crawl/utils/updateFeatures.ts delete mode 100644 apps/server/src/test.ts create mode 100644 apps/server/src/utils/getFilePathDepth.ts delete mode 100644 projects/DeFi/DEX/Coinlink/detail.md delete mode 100644 projects/DeFi/DEX/Coinlink/features.json delete mode 100644 projects/DeFi/DEX/Coinlink/info.json delete mode 100644 projects/DeFi/DEX/Coinlink/logo.png delete mode 100644 projects/DeFi/DEX/Geninus/detail.md delete mode 100644 projects/DeFi/DEX/Geninus/info.json delete mode 100644 projects/DeFi/DEX/Geninus/logo.png delete mode 100644 projects/DeFi/category.json delete mode 100644 projects/DeFi/features.json delete mode 100644 projects/DeFi/glossaries.json delete mode 100644 projects/Wallets/category.json delete mode 100644 projects/socials.json diff --git a/apps/server/src/controllers/crawl.controller.ts b/apps/server/src/controllers/crawl.controller.ts index b5f53e6..df68677 100644 --- a/apps/server/src/controllers/crawl.controller.ts +++ b/apps/server/src/controllers/crawl.controller.ts @@ -4,16 +4,15 @@ import { Request, Response } from 'express' export class CrawlController { constructor(private crawlService = new CrawlService()) {} - public crawl = async (req: Request, res: Response) => { + public crawl = (req: Request, res: Response) => { if (!Object.keys(req.query).includes('dataRaw')) { - res.send('successful') - return + return res.status(200).send('successful') } // @ts-ignore const fileChanges: Array = req.query.dataRaw if (fileChanges.length !== 0) this.crawlService.crawl(fileChanges) - res.send('successful') + return res.status(200).send('successful') } } diff --git a/apps/server/src/crawl.ts b/apps/server/src/crawl.ts index f17db4a..2fd7085 100644 --- a/apps/server/src/crawl.ts +++ b/apps/server/src/crawl.ts @@ -16,7 +16,7 @@ export function fromFileChangesToQuery(fileChanges: Array): string { } export function getURL(data: string) { - const server = 'https://3e71-42-119-180-122.ngrok-free.app' + const server = 'https://44d7-42-119-180-122.ngrok-free.app' return `${server}/api/v1/crawl${data}` } @@ -32,4 +32,4 @@ async function main() { main() -// time for test +// time for demo diff --git a/apps/server/src/crawl/classificationCase.ts b/apps/server/src/crawl/classificationCase.ts index 405f5e2..6482b9e 100644 --- a/apps/server/src/crawl/classificationCase.ts +++ b/apps/server/src/crawl/classificationCase.ts @@ -1,12 +1,19 @@ import 'dotenv/config' import { filterNewParent } from './utils/filterNewParent' import { filterNewSub } from './utils/filterNewSub' -import { filterNewProject } from './utils/filterNewProject' +import { filterNewSocial } from './utils/filterNewSocials' +import { filterNewFeaturesAndGlossaries } from './utils/filterNewFeaturesAndGlossaries' +import { filterProject } from './utils/filterProject' +import { removeTrash } from './utils/removeTrash' export class DataReturn { + isSocialCreate: boolean parentCategory: Array // path + glossaries: Array + features: Array subCategory: Array // path, - project: Array // path + projectUpdate: Array // path + projectDelete: Array // path } export async function classificationCase( @@ -15,9 +22,12 @@ export async function classificationCase( // fileChange is an array of file path const dataReturn = new DataReturn() - const fileLay1 = await filterNewParent(dataReturn, fileChanges) - const fileLay2 = await filterNewSub(dataReturn, fileLay1) - filterNewProject(dataReturn, fileLay2) + const fileLay0 = removeTrash(fileChanges) + const fileLay1 = filterNewSocial(dataReturn, fileLay0) + const fileLay2 = await filterNewParent(dataReturn, fileLay1) + const fileLay3 = filterNewFeaturesAndGlossaries(dataReturn, fileLay2) + const fileLay4 = await filterNewSub(dataReturn, fileLay3) + await filterProject(dataReturn, fileLay4) return dataReturn } diff --git a/apps/server/src/crawl/handleCase.ts b/apps/server/src/crawl/handleCase.ts index 55e4220..503e83d 100644 --- a/apps/server/src/crawl/handleCase.ts +++ b/apps/server/src/crawl/handleCase.ts @@ -9,6 +9,10 @@ import { connection } from '@/databases/connection' import { Categories } from '@/databases/entities/Categories' import { Projects } from '@/databases/entities/Projects' import { ProjectJSON } from '@/shared/schema/ProjectJSON' +import { creatorSocial } from '@/creator/creatorSocial' +import { creatorFeature } from '@/creator/creatorFeature' +import { creatorGlossary } from '@/creator/creatorGlossary' +import { getFileName } from '@/utils/getFileName' function getCategoryName(projectFolder: string): string { const array = projectFolder.split('/') @@ -26,20 +30,40 @@ async function getCategory(projectFolder: string) { } export async function handleCase(caseData: DataReturn) { + if (caseData.isSocialCreate == true) { + creatorSocial() + } + if (caseData.parentCategory.length !== 0) { caseData.parentCategory.map((parent) => { creatorParentCategory(`/${parent}`) }) } + if (caseData.features.length !== 0) { + caseData.features.map(async (filePath) => { + const array = filePath.split('/') + const path = `/${array[0]}/${array[1]}` + await creatorFeature(path) + }) + } + + if (caseData.glossaries.length !== 0) { + caseData.glossaries.map(async (filePath) => { + const array = filePath.split('/') + const path = `/${array[0]}/${array[1]}` + await creatorGlossary(path) + }) + } + if (caseData.subCategory.length !== 0) { caseData.subCategory.map((subPath) => { creatorSubCategory(`/${subPath}`) }) } - if (caseData.project.length !== 0) { - caseData.project.map(async (projectPath) => { + if (caseData.projectUpdate.length !== 0) { + caseData.projectUpdate.map(async (projectPath) => { const detailRaw = await fsWrapper.readFile(`/${projectPath}/info.json`) const detail: ProjectJSON = JSON.parse(detailRaw) @@ -56,4 +80,15 @@ export async function handleCase(caseData: DataReturn) { creatorProject(`/${projectPath}`, category) }) } + + if (caseData.projectDelete.length !== 0) { + caseData.projectDelete.map(async (path) => { + const name = getFileName(path) + const project = await connection + .getRepository(Projects) + .findOneBy({ name }) + + deleteProject(project) + }) + } } diff --git a/apps/server/src/crawl/utils/filterNewFeaturesAndGlossaries.ts b/apps/server/src/crawl/utils/filterNewFeaturesAndGlossaries.ts new file mode 100644 index 0000000..b37d704 --- /dev/null +++ b/apps/server/src/crawl/utils/filterNewFeaturesAndGlossaries.ts @@ -0,0 +1,31 @@ +import { getFilePathDepth } from '@/utils/getFilePathDepth' +import { DataReturn } from '../classificationCase' + +export function filterNewFeaturesAndGlossaries( + dataReturn: DataReturn, + fileChanges: Array, +) { + // Ex of features path: projects/DeFi/features.json + const file1 = fileChanges.filter((value) => getFilePathDepth(value) == 3) + if (file1.length == 0) { + dataReturn.features = [] + dataReturn.glossaries = [] + return fileChanges + } + + dataReturn.features = [] + dataReturn.glossaries = [] + + file1.forEach((value) => { + if (value.includes('features.json')) { + dataReturn.features.push(value) + } else if (value.includes('glossaries.json')) { + dataReturn.glossaries.push(value) + } + }) + + const filesReturn = fileChanges.filter( + (value) => getFilePathDepth(value) !== 3, + ) + return filesReturn +} diff --git a/apps/server/src/crawl/utils/filterNewProject.ts b/apps/server/src/crawl/utils/filterNewProject.ts deleted file mode 100644 index 0c7a65e..0000000 --- a/apps/server/src/crawl/utils/filterNewProject.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { DataReturn } from '../classificationCase' - -function getProjectPath(file: string) { - const array = file.split('/') - - return `${array[0]}/${array[1]}/${array[2]}/${array[3]}` -} - -function getListProject(fileChange: Array): Array { - const set = new Set() - - fileChange.forEach((file) => { - const path = getProjectPath(file) - set.add(path) - }) - - return Array.from(set) -} - -export function filterNewProject( - dataReturn: DataReturn, - fileChange: Array, -) { - if (fileChange.length == 0) { - dataReturn.project = [] - return - } - const projectPaths = getListProject(fileChange) - dataReturn.project = projectPaths -} diff --git a/apps/server/src/crawl/utils/filterNewSocials.ts b/apps/server/src/crawl/utils/filterNewSocials.ts new file mode 100644 index 0000000..2f027fb --- /dev/null +++ b/apps/server/src/crawl/utils/filterNewSocials.ts @@ -0,0 +1,17 @@ +import { DataReturn } from '../classificationCase' + +export function filterNewSocial( + dataReturn: DataReturn, + fileChanges: Array, +): Array { + if (fileChanges.includes('projects/socials.json')) { + dataReturn.isSocialCreate = true + const listFiles = fileChanges.filter( + (value) => value !== 'projects/socials.json', + ) + return listFiles + } + + dataReturn.isSocialCreate = false + return fileChanges +} diff --git a/apps/server/src/crawl/utils/filterProject.ts b/apps/server/src/crawl/utils/filterProject.ts new file mode 100644 index 0000000..fa0ad63 --- /dev/null +++ b/apps/server/src/crawl/utils/filterProject.ts @@ -0,0 +1,47 @@ +import { fsWrapper } from '@/utils/fs/fsWrapper' +import { DataReturn } from '../classificationCase' + +function getProjectPath(file: string) { + const array = file.split('/') + + return `${array[0]}/${array[1]}/${array[2]}/${array[3]}` +} + +function getListProject(fileChange: Array): Array { + const set = new Set() + + fileChange.forEach((file) => { + const path = getProjectPath(file) + set.add(path) + }) + + return Array.from(set) +} + +export async function filterProject( // include new, update (new) and delete project + dataReturn: DataReturn, + fileChanges: Array, +) { + if (fileChanges.length == 0) { + dataReturn.projectUpdate = [] + dataReturn.projectDelete = [] + return + } + const projectPaths = getListProject(fileChanges) + + dataReturn.projectUpdate = [] + dataReturn.projectDelete = [] + + await Promise.all( + projectPaths.map((value) => fsWrapper.checkFileExist(value)), + ).then((values) => { + values.forEach((value, index) => { + if (value == true) { + // that mean project exist => maybe this is an update or new + dataReturn.projectUpdate.push(projectPaths[index]) + } else { + dataReturn.projectDelete.push(projectPaths[index]) + } + }) + }) +} diff --git a/apps/server/src/crawl/utils/removeTrash.ts b/apps/server/src/crawl/utils/removeTrash.ts new file mode 100644 index 0000000..6941deb --- /dev/null +++ b/apps/server/src/crawl/utils/removeTrash.ts @@ -0,0 +1,11 @@ +export function removeTrash(fileChanges: Array) { + if (fileChanges.includes('projects/projects.json')) { + const files = fileChanges.filter( + (value) => value !== 'projects/projects.json', + ) + + return files + } + + return fileChanges +} diff --git a/apps/server/src/crawl/utils/updateFeatures.ts b/apps/server/src/crawl/utils/updateFeatures.ts new file mode 100644 index 0000000..a431cfd --- /dev/null +++ b/apps/server/src/crawl/utils/updateFeatures.ts @@ -0,0 +1,7 @@ +import { fsWrapper } from '@/utils/fs/fsWrapper' + +export async function updateFeatures(filePath: string) { + // ex of filePath: /projects/... + const dataRaw = await fsWrapper.readFile(filePath) + const datas = JSON.parse(dataRaw) +} diff --git a/apps/server/src/test.ts b/apps/server/src/test.ts deleted file mode 100644 index 6bd9c7c..0000000 --- a/apps/server/src/test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { deleteProject } from './crawl/utils/deleteProject' -import { connection } from './databases/connection' -import { Projects } from './databases/entities/Projects' - -async function main() { - await connection.initialize() - - const project = await connection - .getRepository(Projects) - .findOneBy({ name: 'DeFi' }) - - console.log(project) - - deleteProject(project) -} - -main() diff --git a/apps/server/src/utils/fs/fsWrapper.ts b/apps/server/src/utils/fs/fsWrapper.ts index e107d35..6d5592b 100644 --- a/apps/server/src/utils/fs/fsWrapper.ts +++ b/apps/server/src/utils/fs/fsWrapper.ts @@ -48,4 +48,15 @@ export class fsWrapper { }) }) } + + static checkFileExist(path: string): Promise { + return new Promise((res, rej) => { + const request = fsGithubRequest(path) + exec(request, (error, stdout, stderr) => { + const data = JSON.parse(stdout) + if (data.message == 'Not Found') res(false) + else res(true) + }) + }) + } } diff --git a/apps/server/src/utils/getFilePathDepth.ts b/apps/server/src/utils/getFilePathDepth.ts new file mode 100644 index 0000000..618a113 --- /dev/null +++ b/apps/server/src/utils/getFilePathDepth.ts @@ -0,0 +1,4 @@ +export function getFilePathDepth(path: string) { + const array = path.split('/') + return array.length +} diff --git a/projects/DeFi/DEX/Coinlink/detail.md b/projects/DeFi/DEX/Coinlink/detail.md deleted file mode 100644 index e69de29..0000000 diff --git a/projects/DeFi/DEX/Coinlink/features.json b/projects/DeFi/DEX/Coinlink/features.json deleted file mode 100644 index 8672f05..0000000 --- a/projects/DeFi/DEX/Coinlink/features.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - { - "key": "3dArt", - "value": true - } -] \ No newline at end of file diff --git a/projects/DeFi/DEX/Coinlink/info.json b/projects/DeFi/DEX/Coinlink/info.json deleted file mode 100644 index d7ce85a..0000000 --- a/projects/DeFi/DEX/Coinlink/info.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "industry": [ - "Coinlink", - "Dex" - ], - "display_term": "Coinlink", - "term": "defi", - "tags": [ - "AMM", - "CLMM", - "Swaps", - "NFT", - "Liquidity Manager", - "Yield Farming", - "Yield Optimizer" - ], - "glossaries": ["table1"], - "short_description": "Acala provides backend infrastructure for traditional finance that is trusted by institutions like Coinbase, Figment, and Current.com.", - "partnerships": [ - { - "name": "Ledgity", - "image": "https://firebasestorage.googleapis.com/v0/b/builtoncardano.appspot.com/o/uploads%2F1632258485557.svg%2Bxml?alt=media&token=d6d74819-e40d-4e7b-a4be-002653835b01" - }, - { - "name": "Ledgity", - "image": "https://firebasestorage.googleapis.com/v0/b/builtoncardano.appspot.com/o/uploads%2F1632258485557.svg%2Bxml?alt=media&token=d6d74819-e40d-4e7b-a4be-002653835b01" - } - ], - "author": "builtongno", - "social": { - "website": "https://defilama.com", - "twitter": "https://twitter.com/defilama", - "github": "https://github.com/defilama", - "discord": "https://discord.com/invite/defilama", - "telegram": "https://t.me/defilama" - } -} \ No newline at end of file diff --git a/projects/DeFi/DEX/Coinlink/logo.png b/projects/DeFi/DEX/Coinlink/logo.png deleted file mode 100644 index e69de29..0000000 diff --git a/projects/DeFi/DEX/Geninus/detail.md b/projects/DeFi/DEX/Geninus/detail.md deleted file mode 100644 index e69de29..0000000 diff --git a/projects/DeFi/DEX/Geninus/info.json b/projects/DeFi/DEX/Geninus/info.json deleted file mode 100644 index 14d5268..0000000 --- a/projects/DeFi/DEX/Geninus/info.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "industry": [ - "DeFi", - "Dex" - ], - "display_term": "DeFi", - "term": "defi", - "tags": [ - "AMM", - "CLMM", - "Swaps", - "NFT" - ], - "glossaries": ["table1"], - "short_description": "Acala provides backend infrastructure for traditional finance that is trusted by institutions like Coinbase, Figment, and Current.com.", - "partnerships": [ - { - "name": "Ledgity", - "image": "https://firebasestorage.googleapis.com/v0/b/builtoncardano.appspot.com/o/uploads%2F1632258485557.svg%2Bxml?alt=media&token=d6d74819-e40d-4e7b-a4be-002653835b01" - }, - { - "name": "Ledgity", - "image": "https://firebasestorage.googleapis.com/v0/b/builtoncardano.appspot.com/o/uploads%2F1632258485557.svg%2Bxml?alt=media&token=d6d74819-e40d-4e7b-a4be-002653835b01" - } - ], - "author": "builtongno", - "social": { - "website": "https://defilama.com", - "twitter": "https://twitter.com/defilama", - "github": "https://github.com/defilama", - "discord": "https://discord.com/invite/defilama", - "telegram": "https://t.me/defilama" - } -} \ No newline at end of file diff --git a/projects/DeFi/DEX/Geninus/logo.png b/projects/DeFi/DEX/Geninus/logo.png deleted file mode 100644 index e69de29..0000000 diff --git a/projects/DeFi/category.json b/projects/DeFi/category.json deleted file mode 100644 index abc335f..0000000 --- a/projects/DeFi/category.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "DeFi", - "description": "Acala provides backend infrastructure for traditional finance that is trusted by institutions like Coinbase, Figment, and Current.com.", - "pathname": "DeFi", - "status": "active", - "tags": [ - "DeFi", - "Education", - "Videos", - "Merchandise", - "Learn", - "AMM", - "CLMM", - "Swaps", - "NFT", - "Liquidity Manager", - "Yield Farming", - "Yield Optimizer" - ], - "sub_categories": [ - { - "name": "Dex", - "pathname": "DEX", - "status": "active", - "description": "Acala provides backend infrastructure for traditional finance that is trusted by institutions like Coinbase, Figment, and Current.com." - } - ] -} \ No newline at end of file diff --git a/projects/DeFi/features.json b/projects/DeFi/features.json deleted file mode 100644 index 5940410..0000000 --- a/projects/DeFi/features.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "key": "3dArt", - "label": "3D and Art", - "status": "active" - }, - { - "key": "2dAction", - "label": "2D and Action", - "status": "active" - } -] \ No newline at end of file diff --git a/projects/DeFi/glossaries.json b/projects/DeFi/glossaries.json deleted file mode 100644 index 793ce1c..0000000 --- a/projects/DeFi/glossaries.json +++ /dev/null @@ -1,10 +0,0 @@ -[ - { - "name": "table1", - "description": "This is the glossary for table 1" - }, - { - "name": "table2", - "description": "This is the glossary for table 1" - } -] \ No newline at end of file diff --git a/projects/Wallets/category.json b/projects/Wallets/category.json deleted file mode 100644 index 8710f39..0000000 --- a/projects/Wallets/category.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "Wallets", - "description": "Acala provides backend infrastructure for traditional finance that is trusted by institutions like Coinbase, Figment, and Current.com.", - "pathname": "Wallets", - "status": "active", - "tags": [], - "sub_categories": [] -} \ No newline at end of file diff --git a/projects/projects.json b/projects/projects.json index 1acd8e1..7ac6a56 100644 --- a/projects/projects.json +++ b/projects/projects.json @@ -1,22 +1,5 @@ { - "projects":"Built on gno", - "description":"Built on gnosis", - "categories":[ - { - "name":"DeFi", - "pathname":"DeFi" - }, - { - "name":"Wallets", - "pathname":"Wallets" - }, - { - "name":"NFT Collections", - "pathname":"NFT-Collections" - }, - { - "name":"Memecoins", - "pathname":"Memecoins" - } - ] -} \ No newline at end of file + "projects": "Built on gno", + "description": "Built on gnosis", + "categories": [] +} diff --git a/projects/socials.json b/projects/socials.json deleted file mode 100644 index fe540af..0000000 --- a/projects/socials.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "name": "Website", - "code": "website", - "icon_url": "web.com" - }, - { - "name": "Twitter", - "code": "twitter", - "icon_url": "twitter.com" - }, - { - "name": "Github", - "code": "github", - "icon_url": "github.com" - }, - { - "name": "Discord", - "code": "discord", - "icon_url": "discord.com" - }, - { - "name": "Telegram", - "code": "telegram", - "icon_url": "telegram.com" - } -] \ No newline at end of file