DbUrlList now honors recrawlInMs option.

Fixes #49
brendonboshell · Nov 9, 2019 · 0858ef0 · 0858ef0
1 parent 6f85107
commit 0858ef0
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/lib/DbUrlList.js b/lib/DbUrlList.js
@@ -179,7 +179,7 @@ DbUrlList.prototype._calcNextRetryDate = function (numErrors) {
     // If we want to schedule a crawl now, we subtract a random number of
     // seconds. This ensures the order we crawl URLs is random; otherwise, if
     // we parse a sitemap, we could get stuck crawling one host for hours.
-    delay = - Math.random() * YEAR_MS;
+    delay = - Math.random() * this._recrawlInMs;
   } else {
     delay = this._initialRetryTime * Math.pow(2, numErrors - 1);
   }