projects now empty

apps/server/src/controllers/crawl.controller.ts

@@ -4,16 +4,15 @@ import { Request, Response } from 'express'
 export class CrawlController {
   constructor(private crawlService = new CrawlService()) {}
 
-  public crawl = async (req: Request, res: Response) => {
+  public crawl = (req: Request, res: Response) => {
     if (!Object.keys(req.query).includes('dataRaw')) {
-      res.send('successful')
-      return
+      return res.status(200).send('successful')
     }
     // @ts-ignore
     const fileChanges: Array<string> = req.query.dataRaw
 
     if (fileChanges.length !== 0) this.crawlService.crawl(fileChanges)
 
-    res.send('successful')
+    return res.status(200).send('successful')
   }
 }

apps/server/src/crawl.ts

@@ -16,7 +16,7 @@ export function fromFileChangesToQuery(fileChanges: Array<string>): string {
 }
 
 export function getURL(data: string) {
-  const server = 'https://3e71-42-119-180-122.ngrok-free.app'
+  const server = 'https://44d7-42-119-180-122.ngrok-free.app'
 
   return `${server}/api/v1/crawl${data}`
 }
@@ -32,4 +32,4 @@ async function main() {
 
 main()
 
-// time for test
+// time for demo

apps/server/src/crawl/classificationCase.ts

@@ -1,12 +1,19 @@
 import 'dotenv/config'
 import { filterNewParent } from './utils/filterNewParent'
 import { filterNewSub } from './utils/filterNewSub'
-import { filterNewProject } from './utils/filterNewProject'
+import { filterNewSocial } from './utils/filterNewSocials'
+import { filterNewFeaturesAndGlossaries } from './utils/filterNewFeaturesAndGlossaries'
+import { filterProject } from './utils/filterProject'
+import { removeTrash } from './utils/removeTrash'
 
 export class DataReturn {
+  isSocialCreate: boolean
   parentCategory: Array<string> // path
+  glossaries: Array<string>
+  features: Array<string>
   subCategory: Array<string> // path,
-  project: Array<string> // path
+  projectUpdate: Array<string> // path
+  projectDelete: Array<string> // path
 }
 
 export async function classificationCase(
@@ -15,9 +22,12 @@ export async function classificationCase(
   // fileChange is an array of file path
 
   const dataReturn = new DataReturn()
-  const fileLay1 = await filterNewParent(dataReturn, fileChanges)
-  const fileLay2 = await filterNewSub(dataReturn, fileLay1)
-  filterNewProject(dataReturn, fileLay2)
+  const fileLay0 = removeTrash(fileChanges)
+  const fileLay1 = filterNewSocial(dataReturn, fileLay0)
+  const fileLay2 = await filterNewParent(dataReturn, fileLay1)
+  const fileLay3 = filterNewFeaturesAndGlossaries(dataReturn, fileLay2)
+  const fileLay4 = await filterNewSub(dataReturn, fileLay3)
+  await filterProject(dataReturn, fileLay4)
 
   return dataReturn
 }

apps/server/src/crawl/handleCase.ts

@@ -9,6 +9,10 @@ import { connection } from '@/databases/connection'
 import { Categories } from '@/databases/entities/Categories'
 import { Projects } from '@/databases/entities/Projects'
 import { ProjectJSON } from '@/shared/schema/ProjectJSON'
+import { creatorSocial } from '@/creator/creatorSocial'
+import { creatorFeature } from '@/creator/creatorFeature'
+import { creatorGlossary } from '@/creator/creatorGlossary'
+import { getFileName } from '@/utils/getFileName'
 
 function getCategoryName(projectFolder: string): string {
   const array = projectFolder.split('/')
@@ -26,20 +30,40 @@ async function getCategory(projectFolder: string) {
 }
 
 export async function handleCase(caseData: DataReturn) {
+  if (caseData.isSocialCreate == true) {
+    creatorSocial()
+  }
+
   if (caseData.parentCategory.length !== 0) {
     caseData.parentCategory.map((parent) => {
       creatorParentCategory(`/${parent}`)
     })
   }
 
+  if (caseData.features.length !== 0) {
+    caseData.features.map(async (filePath) => {
+      const array = filePath.split('/')
+      const path = `/${array[0]}/${array[1]}`
+      await creatorFeature(path)
+    })
+  }
+
+  if (caseData.glossaries.length !== 0) {
+    caseData.glossaries.map(async (filePath) => {
+      const array = filePath.split('/')
+      const path = `/${array[0]}/${array[1]}`
+      await creatorGlossary(path)
+    })
+  }
+
   if (caseData.subCategory.length !== 0) {
     caseData.subCategory.map((subPath) => {
       creatorSubCategory(`/${subPath}`)
     })
   }
 
-  if (caseData.project.length !== 0) {
-    caseData.project.map(async (projectPath) => {
+  if (caseData.projectUpdate.length !== 0) {
+    caseData.projectUpdate.map(async (projectPath) => {
       const detailRaw = await fsWrapper.readFile(`/${projectPath}/info.json`)
       const detail: ProjectJSON = JSON.parse(detailRaw)
 
@@ -56,4 +80,15 @@ export async function handleCase(caseData: DataReturn) {
       creatorProject(`/${projectPath}`, category)
     })
   }
+
+  if (caseData.projectDelete.length !== 0) {
+    caseData.projectDelete.map(async (path) => {
+      const name = getFileName(path)
+      const project = await connection
+        .getRepository(Projects)
+        .findOneBy({ name })
+
+      deleteProject(project)
+    })
+  }
 }

apps/server/src/crawl/utils/filterNewFeaturesAndGlossaries.ts

@@ -0,0 +1,31 @@
+import { getFilePathDepth } from '@/utils/getFilePathDepth'
+import { DataReturn } from '../classificationCase'
+
+export function filterNewFeaturesAndGlossaries(
+  dataReturn: DataReturn,
+  fileChanges: Array<string>,
+) {
+  // Ex of features path: projects/DeFi/features.json
+  const file1 = fileChanges.filter((value) => getFilePathDepth(value) == 3)
+  if (file1.length == 0) {
+    dataReturn.features = []
+    dataReturn.glossaries = []
+    return fileChanges
+  }
+
+  dataReturn.features = []
+  dataReturn.glossaries = []
+
+  file1.forEach((value) => {
+    if (value.includes('features.json')) {
+      dataReturn.features.push(value)
+    } else if (value.includes('glossaries.json')) {
+      dataReturn.glossaries.push(value)
+    }
+  })
+
+  const filesReturn = fileChanges.filter(
+    (value) => getFilePathDepth(value) !== 3,
+  )
+  return filesReturn
+}

apps/server/src/crawl/utils/filterNewSocials.ts

@@ -0,0 +1,17 @@
+import { DataReturn } from '../classificationCase'
+
+export function filterNewSocial(
+  dataReturn: DataReturn,
+  fileChanges: Array<string>,
+): Array<string> {
+  if (fileChanges.includes('projects/socials.json')) {
+    dataReturn.isSocialCreate = true
+    const listFiles = fileChanges.filter(
+      (value) => value !== 'projects/socials.json',
+    )
+    return listFiles
+  }
+
+  dataReturn.isSocialCreate = false
+  return fileChanges
+}

apps/server/src/crawl/utils/filterProject.ts

@@ -0,0 +1,47 @@
+import { fsWrapper } from '@/utils/fs/fsWrapper'
+import { DataReturn } from '../classificationCase'
+
+function getProjectPath(file: string) {
+  const array = file.split('/')
+
+  return `${array[0]}/${array[1]}/${array[2]}/${array[3]}`
+}
+
+function getListProject(fileChange: Array<string>): Array<string> {
+  const set = new Set<string>()
+
+  fileChange.forEach((file) => {
+    const path = getProjectPath(file)
+    set.add(path)
+  })
+
+  return Array.from(set)
+}
+
+export async function filterProject( // include new, update (new) and delete project
+  dataReturn: DataReturn,
+  fileChanges: Array<string>,
+) {
+  if (fileChanges.length == 0) {
+    dataReturn.projectUpdate = []
+    dataReturn.projectDelete = []
+    return
+  }
+  const projectPaths = getListProject(fileChanges)
+
+  dataReturn.projectUpdate = []
+  dataReturn.projectDelete = []
+
+  await Promise.all(
+    projectPaths.map((value) => fsWrapper.checkFileExist(value)),
+  ).then((values) => {
+    values.forEach((value, index) => {
+      if (value == true) {
+        // that mean project exist => maybe this is an update or new
+        dataReturn.projectUpdate.push(projectPaths[index])
+      } else {
+        dataReturn.projectDelete.push(projectPaths[index])
+      }
+    })
+  })
+}

apps/server/src/crawl/utils/removeTrash.ts

@@ -0,0 +1,11 @@
+export function removeTrash(fileChanges: Array<string>) {
+  if (fileChanges.includes('projects/projects.json')) {
+    const files = fileChanges.filter(
+      (value) => value !== 'projects/projects.json',
+    )
+
+    return files
+  }
+
+  return fileChanges
+}

apps/server/src/crawl/utils/updateFeatures.ts

@@ -0,0 +1,7 @@
+import { fsWrapper } from '@/utils/fs/fsWrapper'
+
+export async function updateFeatures(filePath: string) {
+  // ex of filePath: /projects/...
+  const dataRaw = await fsWrapper.readFile(filePath)
+  const datas = JSON.parse(dataRaw)
+}

apps/server/src/utils/fs/fsWrapper.ts

@@ -48,4 +48,15 @@ export class fsWrapper {
       })
     })
   }
+
+  static checkFileExist(path: string): Promise<boolean> {
+    return new Promise((res, rej) => {
+      const request = fsGithubRequest(path)
+      exec(request, (error, stdout, stderr) => {
+        const data = JSON.parse(stdout)
+        if (data.message == 'Not Found') res(false)
+        else res(true)
+      })
+    })
+  }
 }

apps/server/src/utils/getFilePathDepth.ts

@@ -0,0 +1,4 @@
+export function getFilePathDepth(path: string) {
+  const array = path.split('/')
+  return array.length
+}