diff --git a/package.json b/package.json index 560e313..c09113f 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@nestjs/config": "^3.2.2", "@nestjs/core": "^10.3.8", "@nestjs/platform-fastify": "^10.3.8", + "@nestjs/schedule": "^4.0.2", "@nestjs/swagger": "^7.3.1", "@nestjs/throttler": "^5.1.2", "@prisma/client": "5.14.0", diff --git a/src/app.module.ts b/src/app.module.ts index 13922a2..36947df 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -6,6 +6,7 @@ import {AdminModule} from "./modules/webtoon/admin/admin.module"; import {ThrottlerGuard, ThrottlerModule} from "@nestjs/throttler"; import {APP_GUARD} from "@nestjs/core"; import {MigrationModule} from "./modules/webtoon/migration/migration.module"; +import {ScheduleModule} from "@nestjs/schedule"; @Module({ imports: [ @@ -14,6 +15,7 @@ import {MigrationModule} from "./modules/webtoon/migration/migration.module"; ttl: 60000, limit: 50, }]), + ScheduleModule.forRoot(), TaskModule, WebtoonModule, AdminModule, diff --git a/src/common/utils/models/download-queue.ts b/src/common/utils/models/download-queue.ts new file mode 100644 index 0000000..47165c3 --- /dev/null +++ b/src/common/utils/models/download-queue.ts @@ -0,0 +1,65 @@ +import CachedWebtoonModel from "../../../modules/webtoon/webtoon/models/models/cached-webtoon.model"; +import * as fs from "node:fs"; + +export default class DownloadQueue{ + + queuedDownloads: CachedWebtoonModel[]; + currentDownload: CachedWebtoonModel | undefined; + + constructor(){ + this.queuedDownloads = []; + this.currentDownload = undefined; + } + reset(){ + if(this.currentDownload) + this.queuedDownloads.unshift(this.currentDownload); + } + enqueue(element: CachedWebtoonModel): void{ + if(this.isInQueue(element)) + return; + this.queuedDownloads.push(element); + this.saveQueue(); + } + dequeue(): CachedWebtoonModel | null{ + if (this.isQueueEmpty()) + return null; + this.currentDownload = this.queuedDownloads.shift(); + this.saveQueue(); + return this.currentDownload; + } + isInQueue(element: CachedWebtoonModel): boolean{ + return this.queuedDownloads.find(w => w.title === element.title && w.language === element.language) !== undefined; + } + isQueueEmpty(): boolean{ + return this.queuedDownloads.length === 0; + } + clear(): void{ + this.queuedDownloads = []; + this.currentDownload = undefined; + this.saveQueue(); + } + clearCurrentDownload(): void{ + this.currentDownload = undefined; + this.saveQueue(); + } + getQueue(): CachedWebtoonModel[]{ + return this.queuedDownloads; + } + getCurrentDownload(): CachedWebtoonModel | undefined{ + return this.currentDownload; + } + saveQueue(): void{ + const jsonQueue = JSON.stringify(this, null, 2); + fs.writeFileSync("./.cache/download_queue.json", jsonQueue); + } + static loadQueue(): DownloadQueue{ + if(!fs.existsSync("./.cache/download_queue.json")) + return new DownloadQueue(); + const queueFile: Buffer = fs.readFileSync("./.cache/download_queue.json"); + const queue = JSON.parse(queueFile.toString()); + const webtoonQueue = new DownloadQueue(); + webtoonQueue.queuedDownloads = queue.queuedDownloads; + webtoonQueue.currentDownload = queue.currentDownload; + return webtoonQueue; + } +} diff --git a/src/modules/task/task.module.ts b/src/modules/task/task.module.ts index 4920a49..006d95b 100644 --- a/src/modules/task/task.module.ts +++ b/src/modules/task/task.module.ts @@ -1,8 +1,9 @@ import {Module} from "@nestjs/common"; +import {WebtoonUpdateTask} from "./webtoon_update.task"; +import {WebtoonModule} from "../webtoon/webtoon/webtoon.module"; @Module({ - imports: [], - controllers: [], - providers: [], + providers: [WebtoonUpdateTask], + imports: [WebtoonModule] }) export class TaskModule{} diff --git a/src/modules/task/webtoon_update.task.ts b/src/modules/task/webtoon_update.task.ts new file mode 100644 index 0000000..16acbce --- /dev/null +++ b/src/modules/task/webtoon_update.task.ts @@ -0,0 +1,18 @@ +import {Injectable} from "@nestjs/common"; +import {Cron} from "@nestjs/schedule"; +import {DownloadManagerService} from "../webtoon/webtoon/download-manager.service"; + + +@Injectable() +export class WebtoonUpdateTask{ + + constructor( + private readonly downloadManagerService: DownloadManagerService + ){} + + @Cron("0 0 17 * * *") + async handleCron(){ + // Called every day at 17:00 + this.downloadManagerService.updateAllWebtoons(); + } +} diff --git a/src/modules/webtoon/webtoon/download-manager.service.ts b/src/modules/webtoon/webtoon/download-manager.service.ts index 9249567..6ab73ed 100644 --- a/src/modules/webtoon/webtoon/download-manager.service.ts +++ b/src/modules/webtoon/webtoon/download-manager.service.ts @@ -9,6 +9,7 @@ import WebtoonModel from "./models/models/webtoon.model"; import WebtoonDataModel from "./models/models/webtoon-data.model"; import WebtoonQueue from "../../../common/utils/models/webtoon-queue"; import {HttpStatusCode} from "axios"; +import DownloadQueue from "../../../common/utils/models/download-queue"; @Injectable() export class DownloadManagerService{ @@ -17,17 +18,26 @@ export class DownloadManagerService{ private cacheLoaded: boolean = false; private readonly cachePromise: Promise; - private readonly queue: WebtoonQueue; - private currentDownload: CachedWebtoonModel | undefined; + private readonly downloadQueue: DownloadQueue; constructor( private readonly webtoonParser: WebtoonParserService, private readonly webtoonDatabase: WebtoonDatabaseService, private readonly webtoonDownloader: WebtoonDownloaderService, ){ - this.queue = new WebtoonQueue(); + this.downloadQueue = DownloadQueue.loadQueue(); + // If there are downloads in queue, start download + let downloadInProgress = false; + if(this.downloadQueue.getCurrentDownload() || this.downloadQueue.getQueue().length > 0){ + this.downloadQueue.reset(); + downloadInProgress = true; + } this.cachePromise = this.webtoonParser.loadCache(); - this.cachePromise.then(() => this.cacheLoaded = true); + this.cachePromise.then(() => { + this.cacheLoaded = true + if(downloadInProgress) + this.startDownload().then(() => console.log("Download finished.")); + }); } async awaitCache(): Promise{ @@ -38,12 +48,9 @@ export class DownloadManagerService{ if(!this.cacheLoaded) throw new Error("Cache not loaded."); const webtoonOverview: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoonName, language); - // If webtoon already in queue, do nothing - if(this.queue.getElements().find(w => w.title === webtoonOverview.title)) - return; // If queue is empty, start download - this.queue.enqueue(webtoonOverview); - if(!this.currentDownload) + this.downloadQueue.enqueue(webtoonOverview); + if(!this.downloadQueue.getCurrentDownload()) this.startDownload().then(() => console.log("Download finished.")); } @@ -52,58 +59,59 @@ export class DownloadManagerService{ throw new HttpException("Cache not loaded.", HttpStatusCode.TooEarly); for(const webtoonLanguageName of await this.webtoonDatabase.getWebtoonList()){ const webtoonLanguage: CachedWebtoonModel[] = this.webtoonParser.webtoons[webtoonLanguageName.language]; - this.queue.enqueue(webtoonLanguage.find(w => w.title === webtoonLanguageName.title) as CachedWebtoonModel); + this.downloadQueue.enqueue(webtoonLanguage.find(w => w.title === webtoonLanguageName.title) as CachedWebtoonModel); } - if(!this.currentDownload) + if(!this.downloadQueue.getCurrentDownload()) this.startDownload().then(() => console.log("Download finished.")); } private async startDownload(): Promise{ if(!this.cacheLoaded) throw new HttpException("Cache not loaded.", HttpStatusCode.TooEarly); - while(!this.queue.isEmpty()){ - this.currentDownload = this.queue.dequeue(); - if(!this.currentDownload) + while(!this.downloadQueue.isQueueEmpty()){ + const currentDownload: CachedWebtoonModel = this.downloadQueue.dequeue(); + if(!currentDownload) return; - this.logger.debug(`Downloading ${this.currentDownload.title} (${this.currentDownload.language}).`); - if(!await this.webtoonDatabase.isWebtoonSaved(this.currentDownload.title, this.currentDownload.language)){ - const webtoon: WebtoonModel = await this.webtoonParser.getWebtoonInfos(this.currentDownload); + this.logger.debug(`Downloading ${this.downloadQueue.getCurrentDownload().title} (${this.downloadQueue.getCurrentDownload().language}).`); + if(!await this.webtoonDatabase.isWebtoonSaved(this.downloadQueue.getCurrentDownload().title, this.downloadQueue.getCurrentDownload().language)){ + const webtoon: WebtoonModel = await this.webtoonParser.getWebtoonInfos(this.downloadQueue.getCurrentDownload()); const webtoonData: WebtoonDataModel = await this.webtoonDownloader.downloadWebtoon(webtoon); await this.webtoonDatabase.saveWebtoon(webtoon, webtoonData); } - const startEpisode: number = await this.webtoonDatabase.getLastSavedEpisodeNumber(this.currentDownload.title, this.currentDownload.language); - const epList: EpisodeModel[] = await this.webtoonParser.getEpisodes(this.currentDownload); + const startEpisode: number = await this.webtoonDatabase.getLastSavedEpisodeNumber(this.downloadQueue.getCurrentDownload().title, this.downloadQueue.getCurrentDownload().language); + const epList: EpisodeModel[] = await this.webtoonParser.getEpisodes(this.downloadQueue.getCurrentDownload()); for(let i = startEpisode; i < epList.length; i++){ - if(!this.currentDownload) + if(!this.downloadQueue.getCurrentDownload()) // If current download is cleared, stop downloading break; - const epImageLinks: string[] = await this.webtoonParser.getEpisodeLinks(this.currentDownload, epList[i]); + const epImageLinks: string[] = await this.webtoonParser.getEpisodeLinks(this.downloadQueue.getCurrentDownload(), epList[i]); const episodeData: EpisodeDataModel = await this.webtoonDownloader.downloadEpisode(epList[i], epImageLinks); - if(!this.currentDownload) + if(!this.downloadQueue.getCurrentDownload()) // If current download is cleared, stop downloading break; - await this.webtoonDatabase.saveEpisode(this.currentDownload, epList[i], episodeData); + await this.webtoonDatabase.saveEpisode(this.downloadQueue.getCurrentDownload(), epList[i], episodeData); } } - this.currentDownload = undefined; + this.downloadQueue.clear(); } - getCurrentDownload(): CachedWebtoonModel | undefined{ - if(this.currentDownload) - return this.currentDownload; + getCurrentDownload(): CachedWebtoonModel{ + if(this.downloadQueue.getCurrentDownload()) + return this.downloadQueue.getCurrentDownload(); throw new NotFoundException("No download in progress."); } getDownloadQueue(): CachedWebtoonModel[]{ - if(!this.currentDownload) - throw new NotFoundException("No download in progress.");; - return [this.currentDownload, ...this.queue.getElements()]; + if(!this.downloadQueue.getCurrentDownload() && this.downloadQueue.getQueue().length === 0) + throw new NotFoundException("No download in progress."); + if(this.downloadQueue.getCurrentDownload()) + return [this.downloadQueue.getCurrentDownload(), ...this.downloadQueue.getQueue()]; + return this.downloadQueue.getQueue(); } skipCurrentDownload(): void{ - this.currentDownload = undefined; + this.downloadQueue.clearCurrentDownload(); } clearDownloadQueue(){ - this.queue.clear(); - this.currentDownload = undefined; + this.downloadQueue.clear(); } }