diff --git a/lib/DbUrlList.js b/lib/DbUrlList.js index b2bb35f..6fbdd69 100644 --- a/lib/DbUrlList.js +++ b/lib/DbUrlList.js @@ -179,7 +179,7 @@ DbUrlList.prototype._calcNextRetryDate = function (numErrors) { // If we want to schedule a crawl now, we subtract a random number of // seconds. This ensures the order we crawl URLs is random; otherwise, if // we parse a sitemap, we could get stuck crawling one host for hours. - delay = - Math.random() * YEAR_MS; + delay = - Math.random() * this._recrawlInMs; } else { delay = this._initialRetryTime * Math.pow(2, numErrors - 1); }