diff --git a/docs/bbs.md b/docs/bbs.md
index 9af90a943f..551c512abd 100644
--- a/docs/bbs.md
+++ b/docs/bbs.md
@@ -130,13 +130,19 @@ pageClass: routes
## Dcard
+::: warning 注意
+
+僅能透過台灣 IP 抓取。
+
+:::
+
### 首頁帖子
-
+
### 板塊帖子
-
+
## Discuz
diff --git a/lib/v2/dcard/section.js b/lib/v2/dcard/section.js
index 3b9a0fe7dd..8fa483fd1b 100644
--- a/lib/v2/dcard/section.js
+++ b/lib/v2/dcard/section.js
@@ -1,13 +1,13 @@
-const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
const utils = require('./utils');
module.exports = async (ctx) => {
- const { type, section = 'posts' } = ctx.params;
+ const { type = 'latest', section = 'posts' } = ctx.params;
+ const limit = ctx.query.limit ? Number(ctx.query.limit) : 30;
+ const browser = await require('@/utils/puppeteer')();
let link = `https://www.dcard.tw/f`;
-
let api = `https://www.dcard.tw/service/api/v2`;
-
let title = `Dcard - `;
if (section !== 'posts' && section !== 'popular' && section !== 'latest') {
@@ -15,31 +15,52 @@ module.exports = async (ctx) => {
api += `/forums/${section}`;
title += `${section} - `;
}
-
api += `/posts`;
-
if (type === 'popular') {
link += '?latest=false';
api += '?popular=true';
title += '熱門';
- } else if (type === 'latest' || !type) {
+ } else {
link += '?latest=true';
api += '?popular=false';
title += '最新';
}
- const response = await got(`${api}&limit=30`, {
- headers: {
- 'user-agent': 'dcard-web',
- },
+ const page = await browser.newPage();
+ await page.setRequestInterception(true);
+ page.on('request', (request) => {
+ request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+ });
+ await page.setExtraHTTPHeaders({
+ referer: `https://www.dcard.tw/f/${section}`,
});
- const items = await utils.ProcessFeed(response.data, ctx.cache);
+ await page.goto(`${api}&limit=100`);
+ await page.waitForSelector('body > pre');
+ const response = await page.evaluate(() => document.querySelector('body > pre').innerText);
+ const cookies = await ctx.cache.tryGet('dcard:cookies', async () => await page.cookies(), 3600, false);
+ await page.close();
+
+ const data = JSON.parse(response);
+ const items = data.map((item) => ({
+ title: `「${item.forumName}」${item.title}`,
+ link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`,
+ description: item.excerpt,
+ author: `${item.school || '匿名'}．${item.gender === 'M' ? '男' : '女'}`,
+ pubDate: parseDate(item.createdAt),
+ category: [item.forumName, ...item.topics],
+ forumAlias: item.forumAlias,
+ id: item.id,
+ }));
+
+ // parse fulltext for first `limit` items
+ const result = await utils.ProcessFeed(items, cookies, browser, limit, ctx.cache);
+ await browser.close();
ctx.state.data = {
title,
link,
description: '不想錯過任何有趣的話題嗎？趕快加入我們吧！',
- item: items,
+ item: result,
};
};
diff --git a/lib/v2/dcard/utils.js b/lib/v2/dcard/utils.js
index f795175694..dd67034216 100644
--- a/lib/v2/dcard/utils.js
+++ b/lib/v2/dcard/utils.js
@@ -1,51 +1,48 @@
-const got = require('@/utils/got');
-const { parseDate } = require('@/utils/parse-date');
+const asyncPool = require('tiny-async-pool');
-const ProcessFeed = async (list, cache) => {
- const result = await Promise.all(
- list.map(async (item) => {
- const url = `https://www.dcard.tw/service/api/v2/posts/${item.id}`;
+const ProcessFeed = async (items, cookies, browser, limit, cache) => {
+ let newCookies = [];
+ const result = [];
+ for await (const item of asyncPool(3, items.slice(0, limit), async (i) => {
+ const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`;
+ const content = await cache.tryGet(`dcard:${i.id}`, async () => {
+ let response;
+ // try catch 处理被删除的帖子
+ try {
+ const page = await browser.newPage();
+ await page.setRequestInterception(true);
+ page.on('request', (request) => {
+ request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+ });
+ await page.setExtraHTTPHeaders({
+ referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`,
+ });
+ await page.setCookie(...cookies);
+ await page.goto(url);
+ await page.waitForSelector('body > pre');
+ response = await page.evaluate(() => document.querySelector('body > pre').innerText);
+ newCookies = await page.cookies();
+ await page.close();
- const content = await cache.tryGet(`dcard:${item.id}`, async () => {
- let response;
+ const data = JSON.parse(response);
+ let body = data.content;
+ body = body.replace(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => `
`);
+ body = body.replace(/(?=https?:\/\/).*(??)$/gim, (m) => `${m}`);
+ body = body.replace(/\n/g, '
');
- // try catch 处理被删除的帖子
+ return body;
+ } catch (error) {
+ return '';
+ }
+ });
- try {
- response = await got(url, {
- headers: {
- 'user-agent': 'dcard-web',
- },
- });
-
- let body = response.data.content;
-
- body = body.replace(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => ``);
-
- body = body.replace(/(?=https?:\/\/).*(??)$/gim, (m) => `${m}`);
-
- body = body.replace(/\n/g, '
');
-
- return body;
- } catch (error) {
- return '';
- }
- });
-
- const single = {
- title: `「${item.forumName}」${item.title}`,
- link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`,
- description: content,
- author: item.school || '匿名',
- pubDate: parseDate(item.createdAt),
- category: [item.forumName, ...item.topics],
- };
-
- return single;
- })
- );
-
- return result.filter((f) => f.description);
+ i.description = content;
+ return i;
+ })) {
+ result.push(item);
+ }
+ await cache.set('dcard:cookies', newCookies, 3600);
+ return [...result, ...items.slice(limit)];
};
module.exports = {