feat: make scraper faster, update models

backend/adonisrc.ts

@@ -10,7 +10,11 @@ export default defineConfig({
   | will be scanned automatically from the "./commands" directory.
   |
   */
-  commands: [() => import('@adonisjs/core/commands'), () => import('@adonisjs/lucid/commands'), () => import('adonisjs-scheduler/commands')],
+  commands: [
+    () => import('@adonisjs/core/commands'),
+    () => import('@adonisjs/lucid/commands'),
+    () => import('adonisjs-scheduler/commands'),
+  ],
 
   /*
   |--------------------------------------------------------------------------
@@ -36,7 +40,7 @@ export default defineConfig({
     {
       file: () => import('adonisjs-scheduler/scheduler_provider'),
       environment: ['console'],
-    }
+    },
   ],
 
   /*
@@ -47,10 +51,14 @@ export default defineConfig({
   | List of modules to import before starting the application.
   |
   */
-  preloads: [() => import('#start/routes'), () => import('#start/kernel'), {
-    file: () => import('#start/scheduler'),
-    environment: ['console'],
-  }],
+  preloads: [
+    () => import('#start/routes'),
+    () => import('#start/kernel'),
+    {
+      file: () => import('#start/scheduler'),
+      environment: ['console'],
+    },
+  ],
 
   /*
   |--------------------------------------------------------------------------

backend/app/models/group.ts

@@ -2,7 +2,7 @@ import { DateTime } from 'luxon'
 import { BaseModel, column } from '@adonisjs/lucid/orm'
 export default class Group extends BaseModel {
   @column({ isPrimary: true })
-  declare id: string
+  declare id: number
 
   @column()
   declare name: string
@@ -20,7 +20,7 @@ export default class Group extends BaseModel {
   declare lecturer: string
 
   @column()
-  declare week: '' | 'TP' | 'TN'
+  declare week: '-' | 'TP' | 'TN'
 
   @column()
   declare day: string
@@ -31,6 +31,9 @@ export default class Group extends BaseModel {
   @column()
   declare courseId: string
 
+  @column()
+  declare url: string
+
   @column.dateTime({ autoCreate: true })
   declare createdAt: DateTime
 

backend/app/scrap-registrations/scrap_registrations.ts

@@ -35,9 +35,23 @@ interface Department {
 
 const DEPARTMENTS_URL = 'https://web.usos.pwr.edu.pl/kontroler.php?_action=news/rejestracje/index'
 
-const fetchData = async (url: string) => {
-  const response = await fetch(url)
-  return response
+async function fetchData(url: string, options = {}, timeout = 10000) {
+  const controller = new AbortController()
+  const timeoutId = setTimeout(() => controller.abort(), timeout)
+
+  try {
+    const response = await fetch(url, {
+      ...options,
+      signal: controller.signal,
+    })
+    clearTimeout(timeoutId)
+    return response
+  } catch (error) {
+    if (error.name === 'AbortError') {
+      throw new Error('Request timed out')
+    }
+    throw error
+  }
 }
 
 const scrapDepartments = async () => {
@@ -109,7 +123,8 @@ const scrapCourses = async (registrationUrl: string) => {
   const body = await response.text()
   const $ = cheerio.load(body)
 
-  const courses = $('main#layout-main-content').find('table.wrnav').find('tbody').children('tr')
+  const courses = $('main#layout-main-content').find('table').find('tbody').children('tr')
+
   courses.each((_, element) => {
     const courseUrl = $(element).find('usos-link').find('a').attr('href')
     if (courseUrl !== undefined) {
@@ -266,13 +281,13 @@ const getGroupNumber = (groupInfo: string) => {
   return match ? match[1] : '1'
 }
 
-const checkWeek = (week: string): 'TN' | 'TP' | '' => {
+const checkWeek = (week: string): 'TN' | 'TP' | '-' => {
   if (week.includes('(nieparzyste)')) {
     return 'TN'
   } else if (week.includes('(parzyste)')) {
     return 'TP'
   }
-  return ''
+  return '-'
 }
 
 const checkDay = (day: string) => {

backend/database/migrations/3_create_groups_table.ts

@@ -5,15 +5,16 @@ export default class extends BaseSchema {
 
   async up() {
     this.schema.createTable(this.tableName, (table) => {
-      table.string('id').primary()
+      table.increments('id')
       table.string('name')
       table.time('start_time')
       table.time('end_time')
       table.string('group')
       table.string('lecturer')
-      table.enum('week', ['', 'TN', 'TP'])
+      table.enum('week', ['-', 'TN', 'TP'])
       table.string('day')
       table.string('type')
+      table.string('url')
       table.string('course_id').references('courses.id').onDelete('CASCADE')
       table.timestamp('created_at').defaultTo('NOW()')
       table.timestamp('updated_at')

backend/database/migrations/5_create_schedule_groups_table.ts

@@ -7,7 +7,7 @@ export default class extends BaseSchema {
     this.schema.createTable(this.tableName, (table) => {
       table.increments('id')
       table.integer('schedule_id').references('schedules.id').onDelete('CASCADE')
-      table.string('group_id').references('groups.id').onDelete('CASCADE')
+      table.integer('group_id').references('groups.id').onDelete('CASCADE')
       table.timestamp('created_at')
       table.timestamp('updated_at')
     })

backend/database/seeders/0_department_seeder.ts

@@ -0,0 +1,39 @@
+import Department from '#models/department'
+import { BaseSeeder } from '@adonisjs/lucid/seeders'
+
+export default class extends BaseSeeder {
+  async run() {
+    await Department.createMany([
+      {
+        id: 'W01-ABC-XY-1',
+        name: 'Department of Computer Science',
+        url: 'https://web.usos.pwr.edu.pl/kontroler.php?_action=news/rejestracje/rejJednostki&jed_org_kod=CS',
+      },
+      {
+        id: 'W02-DEF-ZY-2',
+        name: 'Department of Mathematics',
+        url: 'https://web.usos.pwr.edu.pl/kontroler.php?_action=news/rejestracje/rejJednostki&jed_org_kod=MATH',
+      },
+      {
+        id: 'W03-GHI-WX-3',
+        name: 'Department of Physics',
+        url: 'https://web.usos.pwr.edu.pl/kontroler.php?_action=news/rejestracje/rejJednostki&jed_org_kod=PHYS',
+      },
+      {
+        id: 'W04-JKL-VU-4',
+        name: 'Department of Chemistry',
+        url: 'https://web.usos.pwr.edu.pl/kontroler.php?_action=news/rejestracje/rejJednostki&jed_org_kod=CHEM',
+      },
+      {
+        id: 'W05-MNO-TS-5',
+        name: 'Department of Biology',
+        url: 'https://web.usos.pwr.edu.pl/kontroler.php?_action=news/rejestracje/rejJednostki&jed_org_kod=BIO',
+      },
+      {
+        id: 'W06-PQR-RQ-6',
+        name: 'Department of Engineering',
+        url: 'https://web.usos.pwr.edu.pl/kontroler.php?_action=news/rejestracje/rejJednostki&jed_org_kod=ENG',
+      },
+    ])
+  }
+}

backend/database/seeders/1_registration_seeder.ts

@@ -5,18 +5,21 @@ export default class extends BaseSeeder {
   async run() {
     await Registration.createMany([
       {
+        id: '1',
         name: 'W05-EBR-SI-3',
-        departmentId: 'Wydział Elektryczny [W5]',
+        departmentId: 'W01-ABC-XY-1',
         round: 1,
       },
       {
+        id: '2',
         name: 'W13-HWDP-SI-3',
-        departmentId: 'Wydział Matematyki [W13]',
+        departmentId: 'W01-ABC-XY-1',
         round: 2,
       },
       {
+        id: '3',
         name: 'W4-IST-SI-3',
-        departmentId: 'Wydział Informatyki i Telekomunikacji [W4N]',
+        departmentId: 'W05-MNO-TS-5',
         round: 1,
       },
     ])

backend/database/seeders/3_course_seeder.ts

@@ -6,6 +6,10 @@ export default class extends BaseSeeder {
     await Course.createMany([
       { id: 'usos.mathematics.com', name: 'Mathematics 101', registrationId: '1' },
       { id: 'usos.physics.com', name: 'Physics 2', registrationId: '2' },
+      { id: 'usos.chemistry.com', name: 'Chemistry 101', registrationId: '3' },
+      { id: 'usos.biology.com', name: 'Biology 101', registrationId: '4' },
+      { id: 'usos.history.com', name: 'History 101', registrationId: '5' },
+      { id: 'usos.geography.com', name: 'Geography 101', registrationId: '6' },
     ])
   }
 }

backend/database/seeders/4_group_seeder.ts

@@ -21,10 +21,42 @@ export default class extends BaseSeeder {
         endTime: '13:00:00',
         group: '7',
         lecturer: 'Billu The Goat',
-        week: '',
+        week: '-',
         day: 'Wednesday',
         type: 'W',
-        courseId: 'usos.physics.com',
+      },
+      {
+        name: 'Chemistry 303',
+        startTime: '13:15:00',
+        endTime: '15:00:00',
+        group: '3',
+        lecturer: 'Jane Smith',
+        week: 'TN',
+        day: 'Friday',
+        type: 'L',
+        courseId: 'usos.chemistry.com',
+      },
+      {
+        name: 'Biology 404',
+        startTime: '15:15:00',
+        endTime: '17:00:00',
+        group: '2',
+        lecturer: 'Alice Johnson',
+        week: '-',
+        day: 'Tuesday',
+        type: 'W',
+        courseId: 'usos.biology.com',
+      },
+      {
+        name: 'Computer Science 505',
+        startTime: '17:15:00',
+        endTime: '19:00:00',
+        group: '5',
+        lecturer: 'Bob Brown',
+        week: 'TN',
+        day: 'Thursday',
+        type: 'L',
+        courseId: 'usos.compsci.com',
       },
     ])
   }