diff --git a/lib/core/searcher.js b/lib/core/searcher.js index a3242e782..e64b98808 100644 --- a/lib/core/searcher.js +++ b/lib/core/searcher.js @@ -582,7 +582,7 @@ class Searcher { } const content = (await bili.getOneArticleByCv(id) || '').split('推荐文章')[0], - dyids = content.match(/[0-9]{18}/g) || [], + dyids = content.match(/[0-9]{18,}/g) || [], short_ids = content.match(/(?<=b23.tv\/)[a-zA-Z0-9]{7}/g) || [], short_id_set = [...new Set(short_ids)], short_ids_to_dyids = await Promise.all(short_id_set.map(bili.shortDynamicIdToDyid)), @@ -599,38 +599,34 @@ class Searcher { /**遍历某专栏中的dyids */ for (const dyid of dyid_set) { - if (typeof dyid === 'string' - && dyid.length === utils.dyid_length) { + log.info('获取动态', `查看专栏中所提及动态(${dyid}) (${length--})`); + const card = await bili.getOneDynamicByDyid(dyid); - log.info('获取动态', `查看专栏中所提及动态(${dyid}) (${length--})`); - const card = await bili.getOneDynamicByDyid(dyid); + if (card) { + await utils.delay(get_dynamic_detail_wait); - if (card) { - await utils.delay(get_dynamic_detail_wait); - - const parsed_card = parseDynamicCard(card) - , { is_liked } = parsed_card; - - if ( - ((!check_if_duplicated || check_if_duplicated >= 2) - && is_liked) - || ((check_if_duplicated >= 1) - && await d_storage.searchDyid(dyid)) - ) { - log.info('获取动态', `动态(${dyid})已转发过`); - _weight += 1; - } + const parsed_card = parseDynamicCard(card) + , { is_liked } = parsed_card; - if (_weight >= weight && !not_check_article) { - log.warn('获取动态', '1/2动态曾经转过,该专栏或已查看,故中止'); - _dyinfos = []; - break; - } + if ( + ((!check_if_duplicated || check_if_duplicated >= 2) + && is_liked) + || ((check_if_duplicated >= 1) + && await d_storage.searchDyid(dyid)) + ) { + log.info('获取动态', `动态(${dyid})已转发过`); + _weight += 1; + } - _dyinfos.push(parsed_card); + if (_weight >= weight && !not_check_article) { + log.warn('获取动态', '1/2动态曾经转过,该专栏或已查看,故中止'); + _dyinfos = []; + break; } + + _dyinfos.push(parsed_card); } else { - log.warn('获取动态', `动态(${dyid})无效 (${length--})`); + log.warn('获取动态', `动态细节获取失败(${dyid})`); } } dyinfos.push(..._dyinfos); @@ -751,33 +747,30 @@ class Searcher { dyinfos = []; for (const dyid of dyids) { - if (typeof dyid === 'string' - && dyid.length === utils.dyid_length) { - - log.info('获取动态', `查看Txt中所提及动态(${dyid}) (${length--})`); - const card = await bili.getOneDynamicByDyid(dyid); - - if (card) { - await utils.delay(get_dynamic_detail_wait); - - const parsed_card = parseDynamicCard(card) - , { is_liked } = parsed_card; - - if ( - ((!check_if_duplicated || check_if_duplicated >= 2) - && is_liked) - || ((check_if_duplicated >= 1) - && await d_storage.searchDyid(dyid)) - ) { - log.info('获取动态', `动态(${dyid})已转发过`); - continue; - } - - dyinfos.push(parsed_card); + log.info('获取动态', `查看Txt中所提及动态(${dyid}) (${length--})`); + const card = await bili.getOneDynamicByDyid(dyid); + + if (card) { + await utils.delay(get_dynamic_detail_wait); + + const parsed_card = parseDynamicCard(card) + , { is_liked } = parsed_card; + + if ( + ((!check_if_duplicated || check_if_duplicated >= 2) + && is_liked) + || ((check_if_duplicated >= 1) + && await d_storage.searchDyid(dyid)) + ) { + log.info('获取动态', `动态(${dyid})已转发过`); + continue; } + + dyinfos.push(parsed_card); } else { - log.warn('获取动态', `动态(${dyid})无效 (${length--})`); + log.warn('获取动态', `动态细节获取失败(${dyid})`); } + } const fomatdata = dyinfos.map(o => { return { diff --git a/lib/helper/d_storage.js b/lib/helper/d_storage.js index 46c54faf1..3f308ed3b 100644 --- a/lib/helper/d_storage.js +++ b/lib/helper/d_storage.js @@ -1,4 +1,4 @@ -const { log, readDyidFile, writeDyidFile, dyid_length } = require('../utils'); +const { log, readDyidFile, writeDyidFile } = require('../utils'); const d_storage = { /** @@ -6,24 +6,35 @@ const d_storage = { * @param {string} dyid * @returns {Promise} */ - searchDyid: (dyid) => { - return new Promise((resolve) => { - const Rdyid = new RegExp(dyid); - const rs = readDyidFile(Number(process.env.NUMBER)); - let status = false; - rs.on('data', chunk => { - if (Rdyid.test(chunk)) { - status = true; - } - }); - rs.on('end', () => { - resolve(status); - }); - rs.on('error', err => { - log.error('搜索dyid', err); - resolve(status); - }); - }); + searchDyid: async (dyid) => { + let buffer = ''; + let found = false; + + const stream = readDyidFile(Number(process.env.NUMBER)); + + for await (const chunk of stream) { + buffer += chunk; + + while ((buffer.indexOf(dyid)) !== -1) { + found = true; + stream.destroy(); + return found; + } + + const lastCommaIdx = buffer.lastIndexOf(','); + if (lastCommaIdx !== -1) { + buffer = buffer.slice(lastCommaIdx); + } + } + + if (!found && buffer.length > 0) { + if (buffer.includes(dyid)) { + found = true; + } + } + + stream.destroy(); + return found; }, /** * 更新dyid @@ -31,9 +42,6 @@ const d_storage = { */ updateDyid: (dyid) => { log.info('更新dyid', `写入${dyid}`); - if (dyid.length !== dyid_length) { - log.error('更新dyid', `dyid(${dyid})长度不为18 若出现此问题请即时通知开发者`); - } return new Promise((resolve) => { const ws = writeDyidFile(Number(process.env.NUMBER)); ws.write(dyid + ',', () => { diff --git a/lib/net/bili.js b/lib/net/bili.js index 47ee6fd1d..0cc02a53b 100644 --- a/lib/net/bili.js +++ b/lib/net/bili.js @@ -407,7 +407,7 @@ const bili_client = { redirect: false, } }).then(a => { - const dyid = (a.match(/[0-9]{18}/) || [])[0]; + const dyid = (a.match(/[0-9]{18,}/) || [])[0]; log.info('短连接转换', `${short_id} -> ${dyid}`); return dyid; }); diff --git a/lib/utils.js b/lib/utils.js index 9baad5212..c82f856ba 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -19,8 +19,6 @@ const utils = { lottery_info_dir: path.join(process.cwd(), 'lottery_info'), /**本地抽奖信息存放目录 */ lottery_dyids: path.join(process.cwd(), 'lottery_dyids'), - /**dyid长度 */ - dyid_length: 18, /** * 将版本号转为数字 * @example @@ -402,7 +400,7 @@ const utils = { */ readDyidFile(num) { const fpath = num < 2 ? path.join(utils.dyids_dir, 'dyid.txt') : path.join(utils.dyids_dir, `dyid${num}.txt`); - return fs.createReadStream(fpath, { encoding: 'utf8', highWaterMark: (utils.dyid_length + 1) * 1000 }); + return fs.createReadStream(fpath, { encoding: 'utf8' }); }, /** * 追加dyid diff --git a/test/api.test.js b/test/api.test.js index 145a02c8b..59f745e69 100644 --- a/test/api.test.js +++ b/test/api.test.js @@ -6,7 +6,7 @@ const { parseDynamicCard } = require('../lib/core/searcher'); (async () => { assert.notEqual(await bili_client.getMyinfo(), null); - await util.par_run([0, 3], [ + await util.par_run([], [ // 0 async () => { assert.equal((await bili_client.getTopRcmd()).length, 10); diff --git a/test/dyidSearch.test.js b/test/dyidSearch.test.js new file mode 100644 index 000000000..f493c0ac0 --- /dev/null +++ b/test/dyidSearch.test.js @@ -0,0 +1,14 @@ +const assert = require('assert'); +const util = require('./util'); +const d_storage = require('../lib/helper/d_storage'); + +(async () => { + await util.par_run([0], [ + // 0 + async () => { + assert(await d_storage.searchDyid('1234567901234568')); + assert(!await d_storage.searchDyid('1234567901234569')); + }, + ]); + console.log('dyidSearch.test ... ok!'); +})(); \ No newline at end of file diff --git a/test/dynamic_card.test.js b/test/dynamic_card.test.js index 7d37c9bea..010599faf 100644 --- a/test/dynamic_card.test.js +++ b/test/dynamic_card.test.js @@ -4,7 +4,7 @@ const searcher = require('../lib/core/searcher'); const util = require('./util'); (async () => { - await util.par_run([0, 1, 2, 3, 4, 5, 6, 7, 8], [ + await util.par_run([], [ // 0 async () => { let info = await bili_client.getOneDynamicByDyid('728424890210713624');