Merge pull request #31 from Open-Webtoon-Reader/feature/better-downloads

Merge better downloads to Main
Open-Webtoon-Reader · May 28, 2024 · cacd8b3 · cacd8b3
2 parents d8067dd + 7655868
commit cacd8b3
Show file tree

Hide file tree

Showing 6 changed files with 131 additions and 36 deletions.
diff --git a/package.json b/package.json
@@ -24,6 +24,7 @@
     "@nestjs/config": "^3.2.2",
     "@nestjs/core": "^10.3.8",
     "@nestjs/platform-fastify": "^10.3.8",
+    "@nestjs/schedule": "^4.0.2",
     "@nestjs/swagger": "^7.3.1",
     "@nestjs/throttler": "^5.1.2",
     "@prisma/client": "5.14.0",

diff --git a/src/app.module.ts b/src/app.module.ts
@@ -6,6 +6,7 @@ import {AdminModule} from "./modules/webtoon/admin/admin.module";
 import {ThrottlerGuard, ThrottlerModule} from "@nestjs/throttler";
 import {APP_GUARD} from "@nestjs/core";
 import {MigrationModule} from "./modules/webtoon/migration/migration.module";
+import {ScheduleModule} from "@nestjs/schedule";
 
 @Module({
     imports: [
@@ -14,6 +15,7 @@ import {MigrationModule} from "./modules/webtoon/migration/migration.module";
             ttl: 60000,
             limit: 50,
         }]),
+        ScheduleModule.forRoot(),
         TaskModule,
         WebtoonModule,
         AdminModule,

diff --git a/src/common/utils/models/download-queue.ts b/src/common/utils/models/download-queue.ts
@@ -0,0 +1,65 @@
+import CachedWebtoonModel from "../../../modules/webtoon/webtoon/models/models/cached-webtoon.model";
+import * as fs from "node:fs";
+
+export default class DownloadQueue{
+
+    queuedDownloads: CachedWebtoonModel[];
+    currentDownload: CachedWebtoonModel | undefined;
+
+    constructor(){
+        this.queuedDownloads = [];
+        this.currentDownload = undefined;
+    }
+    reset(){
+        if(this.currentDownload)
+            this.queuedDownloads.unshift(this.currentDownload);
+    }
+    enqueue(element: CachedWebtoonModel): void{
+        if(this.isInQueue(element))
+            return;
+        this.queuedDownloads.push(element);
+        this.saveQueue();
+    }
+    dequeue(): CachedWebtoonModel | null{
+        if (this.isQueueEmpty())
+            return null;
+        this.currentDownload = this.queuedDownloads.shift();
+        this.saveQueue();
+        return this.currentDownload;
+    }
+    isInQueue(element: CachedWebtoonModel): boolean{
+        return this.queuedDownloads.find(w => w.title === element.title && w.language === element.language) !== undefined;
+    }
+    isQueueEmpty(): boolean{
+        return this.queuedDownloads.length === 0;
+    }
+    clear(): void{
+        this.queuedDownloads = [];
+        this.currentDownload = undefined;
+        this.saveQueue();
+    }
+    clearCurrentDownload(): void{
+        this.currentDownload = undefined;
+        this.saveQueue();
+    }
+    getQueue(): CachedWebtoonModel[]{
+        return this.queuedDownloads;
+    }
+    getCurrentDownload(): CachedWebtoonModel | undefined{
+        return this.currentDownload;
+    }
+    saveQueue(): void{
+        const jsonQueue = JSON.stringify(this, null, 2);
+        fs.writeFileSync("./.cache/download_queue.json", jsonQueue);
+    }
+    static loadQueue(): DownloadQueue{
+        if(!fs.existsSync("./.cache/download_queue.json"))
+            return new DownloadQueue();
+        const queueFile: Buffer = fs.readFileSync("./.cache/download_queue.json");
+        const queue = JSON.parse(queueFile.toString());
+        const webtoonQueue = new DownloadQueue();
+        webtoonQueue.queuedDownloads = queue.queuedDownloads;
+        webtoonQueue.currentDownload = queue.currentDownload;
+        return webtoonQueue;
+    }
+}
diff --git a/src/modules/task/task.module.ts b/src/modules/task/task.module.ts
@@ -1,8 +1,9 @@
 import {Module} from "@nestjs/common";
+import {WebtoonUpdateTask} from "./webtoon_update.task";
+import {WebtoonModule} from "../webtoon/webtoon/webtoon.module";
 
 @Module({
-    imports: [],
-    controllers: [],
-    providers: [],
+    providers: [WebtoonUpdateTask],
+    imports: [WebtoonModule]
 })
 export class TaskModule{}
diff --git a/src/modules/task/webtoon_update.task.ts b/src/modules/task/webtoon_update.task.ts
@@ -0,0 +1,18 @@
+import {Injectable} from "@nestjs/common";
+import {Cron} from "@nestjs/schedule";
+import {DownloadManagerService} from "../webtoon/webtoon/download-manager.service";
+
+
+@Injectable()
+export class WebtoonUpdateTask{
+
+    constructor(
+        private readonly downloadManagerService: DownloadManagerService
+    ){}
+
+    @Cron("0 0 17 * * *")
+    async handleCron(){
+        // Called every day at 17:00
+        this.downloadManagerService.updateAllWebtoons();
+    }
+}
diff --git a/src/modules/webtoon/webtoon/download-manager.service.ts b/src/modules/webtoon/webtoon/download-manager.service.ts
@@ -9,6 +9,7 @@ import WebtoonModel from "./models/models/webtoon.model";
 import WebtoonDataModel from "./models/models/webtoon-data.model";
 import WebtoonQueue from "../../../common/utils/models/webtoon-queue";
 import {HttpStatusCode} from "axios";
+import DownloadQueue from "../../../common/utils/models/download-queue";
 
 @Injectable()
 export class DownloadManagerService{
@@ -17,17 +18,26 @@ export class DownloadManagerService{
 
     private cacheLoaded: boolean = false;
     private readonly cachePromise: Promise<void>;
-    private readonly queue: WebtoonQueue;
-    private currentDownload: CachedWebtoonModel | undefined;
+    private readonly downloadQueue: DownloadQueue;
 
     constructor(
         private readonly webtoonParser: WebtoonParserService,
         private readonly webtoonDatabase: WebtoonDatabaseService,
         private readonly webtoonDownloader: WebtoonDownloaderService,
     ){
-        this.queue = new WebtoonQueue();
+        this.downloadQueue = DownloadQueue.loadQueue();
+        // If there are downloads in queue, start download
+        let downloadInProgress = false;
+        if(this.downloadQueue.getCurrentDownload() || this.downloadQueue.getQueue().length > 0){
+            this.downloadQueue.reset();
+            downloadInProgress = true;
+        }
         this.cachePromise = this.webtoonParser.loadCache();
-        this.cachePromise.then(() => this.cacheLoaded = true);
+        this.cachePromise.then(() => {
+            this.cacheLoaded = true
+            if(downloadInProgress)
+                this.startDownload().then(() => console.log("Download finished."));
+        });
     }
 
     async awaitCache(): Promise<void>{
@@ -38,12 +48,9 @@ export class DownloadManagerService{
         if(!this.cacheLoaded)
             throw new Error("Cache not loaded.");
         const webtoonOverview: CachedWebtoonModel = this.webtoonParser.findWebtoon(webtoonName, language);
-        // If webtoon already in queue, do nothing
-        if(this.queue.getElements().find(w => w.title === webtoonOverview.title))
-            return;
         // If queue is empty, start download
-        this.queue.enqueue(webtoonOverview);
-        if(!this.currentDownload)
+        this.downloadQueue.enqueue(webtoonOverview);
+        if(!this.downloadQueue.getCurrentDownload())
             this.startDownload().then(() => console.log("Download finished."));
     }
 
@@ -52,58 +59,59 @@ export class DownloadManagerService{
             throw new HttpException("Cache not loaded.", HttpStatusCode.TooEarly);
         for(const webtoonLanguageName of await this.webtoonDatabase.getWebtoonList()){
             const webtoonLanguage: CachedWebtoonModel[] = this.webtoonParser.webtoons[webtoonLanguageName.language];
-            this.queue.enqueue(webtoonLanguage.find(w => w.title === webtoonLanguageName.title) as CachedWebtoonModel);
+            this.downloadQueue.enqueue(webtoonLanguage.find(w => w.title === webtoonLanguageName.title) as CachedWebtoonModel);
         }
-        if(!this.currentDownload)
+        if(!this.downloadQueue.getCurrentDownload())
             this.startDownload().then(() => console.log("Download finished."));
     }
 
     private async startDownload(): Promise<void>{
         if(!this.cacheLoaded)
             throw new HttpException("Cache not loaded.", HttpStatusCode.TooEarly);
-        while(!this.queue.isEmpty()){
-            this.currentDownload = this.queue.dequeue();
-            if(!this.currentDownload)
+        while(!this.downloadQueue.isQueueEmpty()){
+            const currentDownload: CachedWebtoonModel = this.downloadQueue.dequeue();
+            if(!currentDownload)
                 return;
-            this.logger.debug(`Downloading ${this.currentDownload.title} (${this.currentDownload.language}).`);
-            if(!await this.webtoonDatabase.isWebtoonSaved(this.currentDownload.title, this.currentDownload.language)){
-                const webtoon: WebtoonModel = await this.webtoonParser.getWebtoonInfos(this.currentDownload);
+            this.logger.debug(`Downloading ${this.downloadQueue.getCurrentDownload().title} (${this.downloadQueue.getCurrentDownload().language}).`);
+            if(!await this.webtoonDatabase.isWebtoonSaved(this.downloadQueue.getCurrentDownload().title, this.downloadQueue.getCurrentDownload().language)){
+                const webtoon: WebtoonModel = await this.webtoonParser.getWebtoonInfos(this.downloadQueue.getCurrentDownload());
                 const webtoonData: WebtoonDataModel = await this.webtoonDownloader.downloadWebtoon(webtoon);
                 await this.webtoonDatabase.saveWebtoon(webtoon, webtoonData);
             }
-            const startEpisode: number = await this.webtoonDatabase.getLastSavedEpisodeNumber(this.currentDownload.title, this.currentDownload.language);
-            const epList: EpisodeModel[] = await this.webtoonParser.getEpisodes(this.currentDownload);
+            const startEpisode: number = await this.webtoonDatabase.getLastSavedEpisodeNumber(this.downloadQueue.getCurrentDownload().title, this.downloadQueue.getCurrentDownload().language);
+            const epList: EpisodeModel[] = await this.webtoonParser.getEpisodes(this.downloadQueue.getCurrentDownload());
             for(let i = startEpisode; i < epList.length; i++){
-                if(!this.currentDownload)
+                if(!this.downloadQueue.getCurrentDownload()) // If current download is cleared, stop downloading
                     break;
-                const epImageLinks: string[] = await this.webtoonParser.getEpisodeLinks(this.currentDownload, epList[i]);
+                const epImageLinks: string[] = await this.webtoonParser.getEpisodeLinks(this.downloadQueue.getCurrentDownload(), epList[i]);
                 const episodeData: EpisodeDataModel = await this.webtoonDownloader.downloadEpisode(epList[i], epImageLinks);
-                if(!this.currentDownload)
+                if(!this.downloadQueue.getCurrentDownload()) // If current download is cleared, stop downloading
                     break;
-                await this.webtoonDatabase.saveEpisode(this.currentDownload, epList[i], episodeData);
+                await this.webtoonDatabase.saveEpisode(this.downloadQueue.getCurrentDownload(), epList[i], episodeData);
             }
         }
-        this.currentDownload = undefined;
+        this.downloadQueue.clear();
     }
 
-    getCurrentDownload(): CachedWebtoonModel | undefined{
-        if(this.currentDownload)
-            return this.currentDownload;
+    getCurrentDownload(): CachedWebtoonModel{
+        if(this.downloadQueue.getCurrentDownload())
+            return this.downloadQueue.getCurrentDownload();
         throw new NotFoundException("No download in progress.");
     }
 
     getDownloadQueue(): CachedWebtoonModel[]{
-        if(!this.currentDownload)
-            throw new NotFoundException("No download in progress.");;
-        return [this.currentDownload, ...this.queue.getElements()];
+        if(!this.downloadQueue.getCurrentDownload() && this.downloadQueue.getQueue().length === 0)
+            throw new NotFoundException("No download in progress.");
+        if(this.downloadQueue.getCurrentDownload())
+            return [this.downloadQueue.getCurrentDownload(), ...this.downloadQueue.getQueue()];
+        return this.downloadQueue.getQueue();
     }
 
     skipCurrentDownload(): void{
-        this.currentDownload = undefined;
+        this.downloadQueue.clearCurrentDownload();
     }
 
     clearDownloadQueue(){
-        this.queue.clear();
-        this.currentDownload = undefined;
+        this.downloadQueue.clear();
     }
 }