From ba2df5d3d91f84356d46c9f6c79a3eecbead2ec7 Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 8 Sep 2022 23:55:42 +0700 Subject: [PATCH] fix(route): dcard (#10739) * fix: dcard use puppeteer * docs: add pptr attr * fix: use cookies * docs: add warning --- docs/bbs.md | 10 ++++- lib/v2/dcard/section.js | 47 ++++++++++++++++------- lib/v2/dcard/utils.js | 85 ++++++++++++++++++++--------------------- 3 files changed, 83 insertions(+), 59 deletions(-) diff --git a/docs/bbs.md b/docs/bbs.md index 9af90a943f..551c512abd 100644 --- a/docs/bbs.md +++ b/docs/bbs.md @@ -130,13 +130,19 @@ pageClass: routes ## Dcard +::: warning 注意 + +僅能透過台灣 IP 抓取。 + +::: + ### 首頁帖子 - + ### 板塊帖子 - + ## Discuz diff --git a/lib/v2/dcard/section.js b/lib/v2/dcard/section.js index 3b9a0fe7dd..8fa483fd1b 100644 --- a/lib/v2/dcard/section.js +++ b/lib/v2/dcard/section.js @@ -1,13 +1,13 @@ -const got = require('@/utils/got'); +const { parseDate } = require('@/utils/parse-date'); const utils = require('./utils'); module.exports = async (ctx) => { - const { type, section = 'posts' } = ctx.params; + const { type = 'latest', section = 'posts' } = ctx.params; + const limit = ctx.query.limit ? Number(ctx.query.limit) : 30; + const browser = await require('@/utils/puppeteer')(); let link = `https://www.dcard.tw/f`; - let api = `https://www.dcard.tw/service/api/v2`; - let title = `Dcard - `; if (section !== 'posts' && section !== 'popular' && section !== 'latest') { @@ -15,31 +15,52 @@ module.exports = async (ctx) => { api += `/forums/${section}`; title += `${section} - `; } - api += `/posts`; - if (type === 'popular') { link += '?latest=false'; api += '?popular=true'; title += '熱門'; - } else if (type === 'latest' || !type) { + } else { link += '?latest=true'; api += '?popular=false'; title += '最新'; } - const response = await got(`${api}&limit=30`, { - headers: { - 'user-agent': 'dcard-web', - }, + const page = await browser.newPage(); + await page.setRequestInterception(true); + page.on('request', (request) => { + request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort(); + }); + await page.setExtraHTTPHeaders({ + referer: `https://www.dcard.tw/f/${section}`, }); - const items = await utils.ProcessFeed(response.data, ctx.cache); + await page.goto(`${api}&limit=100`); + await page.waitForSelector('body > pre'); + const response = await page.evaluate(() => document.querySelector('body > pre').innerText); + const cookies = await ctx.cache.tryGet('dcard:cookies', async () => await page.cookies(), 3600, false); + await page.close(); + + const data = JSON.parse(response); + const items = data.map((item) => ({ + title: `「${item.forumName}」${item.title}`, + link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`, + description: item.excerpt, + author: `${item.school || '匿名'}．${item.gender === 'M' ? '男' : '女'}`, + pubDate: parseDate(item.createdAt), + category: [item.forumName, ...item.topics], + forumAlias: item.forumAlias, + id: item.id, + })); + + // parse fulltext for first `limit` items + const result = await utils.ProcessFeed(items, cookies, browser, limit, ctx.cache); + await browser.close(); ctx.state.data = { title, link, description: '不想錯過任何有趣的話題嗎？趕快加入我們吧！', - item: items, + item: result, }; }; diff --git a/lib/v2/dcard/utils.js b/lib/v2/dcard/utils.js index f795175694..dd67034216 100644 --- a/lib/v2/dcard/utils.js +++ b/lib/v2/dcard/utils.js @@ -1,51 +1,48 @@ -const got = require('@/utils/got'); -const { parseDate } = require('@/utils/parse-date'); +const asyncPool = require('tiny-async-pool'); -const ProcessFeed = async (list, cache) => { - const result = await Promise.all( - list.map(async (item) => { - const url = `https://www.dcard.tw/service/api/v2/posts/${item.id}`; +const ProcessFeed = async (items, cookies, browser, limit, cache) => { + let newCookies = []; + const result = []; + for await (const item of asyncPool(3, items.slice(0, limit), async (i) => { + const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`; + const content = await cache.tryGet(`dcard:${i.id}`, async () => { + let response; + // try catch 处理被删除的帖子 + try { + const page = await browser.newPage(); + await page.setRequestInterception(true); + page.on('request', (request) => { + request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort(); + }); + await page.setExtraHTTPHeaders({ + referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`, + }); + await page.setCookie(...cookies); + await page.goto(url); + await page.waitForSelector('body > pre'); + response = await page.evaluate(() => document.querySelector('body > pre').innerText); + newCookies = await page.cookies(); + await page.close(); - const content = await cache.tryGet(`dcard:${item.id}`, async () => { - let response; + const data = JSON.parse(response); + let body = data.content; + body = body.replace(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => ``); + body = body.replace(/(?=https?:\/\/).*(??)$/gim, (m) => `${m}`); + body = body.replace(/\n/g, '
'); - // try catch 处理被删除的帖子 + return body; + } catch (error) { + return ''; + } + }); - try { - response = await got(url, { - headers: { - 'user-agent': 'dcard-web', - }, - }); - - let body = response.data.content; - - body = body.replace(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => ``); - - body = body.replace(/(?=https?:\/\/).*(??)$/gim, (m) => `${m}`); - - body = body.replace(/\n/g, '
'); - - return body; - } catch (error) { - return ''; - } - }); - - const single = { - title: `「${item.forumName}」${item.title}`, - link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`, - description: content, - author: item.school || '匿名', - pubDate: parseDate(item.createdAt), - category: [item.forumName, ...item.topics], - }; - - return single; - }) - ); - - return result.filter((f) => f.description); + i.description = content; + return i; + })) { + result.push(item); + } + await cache.set('dcard:cookies', newCookies, 3600); + return [...result, ...items.slice(limit)]; }; module.exports = {