feat(route/douban): follow the last page of long group topics

Douban lists replies from oldest to newest, 100 per page, so the route
only ever saw the first 100 replies of a long topic and missed every
new one. When the topic has more than one page, fetch the page that the
paginator's last link points to and use its replies instead. Reply
links now open that page (with author=1 in author-only mode), while the
guid keeps the old format so existing items are not duplicated.

Closes #17743

lib/routes/douban/other/group-topic.ts

@@ -1,3 +1,4 @@
+import type { CheerioAPI } from 'cheerio';
 import { load } from 'cheerio';
 
 import { config } from '@/config';
@@ -27,12 +28,29 @@ export const route: Route = {
     },
     features: { antiCrawler: true, requireConfig: [{ name: 'DOUBAN_COOKIE', optional: true, description: '需要登录才能查看的帖子请配置本人豆瓣 Cookie。' }] },
     radar: [{ source: ['www.douban.com/group/topic/:id'], target: '/group/topic/:id' }],
-    description:
-        '订阅主帖正文和源页面首屏回帖。主帖标题或正文改变时产生新 GUID，以供阅读器识别更新。只看楼主使用源站的 author=1 页面。源站回帖按从早到晚排序，因此长帖尾页的新回复尚不在本路由范围内。日期保留源站创建时间，不冒充最后更新时间。',
+    description: '订阅主帖正文和最新一页回帖。源站回帖按从早到晚排序，多页长帖取尾页的回帖。主帖标题或正文改变时产生新 GUID，以供阅读器识别更新。只看楼主使用源站的 author=1 页面。日期保留源站创建时间，不冒充最后更新时间。',
     handler,
 };
 
-export function parseGroupTopic(html: string, link: string): { title: string; item: DataItem[] } {
+function parseReplies($: CheerioAPI, link: string, pageUrl = link): DataItem[] {
+    return $('#comments > .comment-item[id]')
+        .toArray()
+        .map((element): DataItem => {
+            const $reply = $(element);
+            const pubtime = /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})(?:\s+(\S.*))?$/.exec($reply.find('.pubtime').first().text());
+            return {
+                title: $reply.find('.reply-content').text(),
+                description: $reply.find('.reply-content').html(),
+                link: `${pageUrl}#${$reply.attr('id')}`,
+                guid: `${link}#${$reply.attr('id')}`,
+                author: $reply.find('h4 > a').first().text(),
+                pubDate: pubtime ? timezone(parseDate(pubtime[1], 'YYYY-MM-DD HH:mm:ss'), 8) : undefined,
+                category: pubtime?.[2] ? [`IP属地：${pubtime[2]}`] : undefined,
+            };
+        });
+}
+
+export function parseGroupTopic(html: string, link: string): { title: string; item: DataItem[]; lastPageStart?: string } {
     const $ = load(html);
     const title = $('h1').text();
     const content = $('.topic-content .rich-content').html();
@@ -50,21 +68,10 @@ export function parseGroupTopic(html: string, link: string): { title: string; it
         category: ipLocation ? [`IP属地：${ipLocation}`] : undefined,
         guid: `${link}#content-${md5(`${title}\n${content}`)}`,
     };
-    const replies = $('#comments > .comment-item[id]')
-        .toArray()
-        .map((element): DataItem => {
-            const $reply = $(element);
-            const pubtime = /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})(?:\s+(\S.*))?$/.exec($reply.find('.pubtime').first().text());
-            return {
-                title: $reply.find('.reply-content').text(),
-                description: $reply.find('.reply-content').html(),
-                link: `${link}#${$reply.attr('id')}`,
-                author: $reply.find('h4 > a').first().text(),
-                pubDate: pubtime ? timezone(parseDate(pubtime[1], 'YYYY-MM-DD HH:mm:ss'), 8) : undefined,
-                category: pubtime?.[2] ? [`IP属地：${pubtime[2]}`] : undefined,
-            };
-        });
-    return { title, item: [post, ...replies] };
+    // The last direct link of the paginator points to the last page
+    const lastPage = $('.paginator > a').last().attr('href');
+    const lastPageStart = lastPage ? (new URL(lastPage, link).searchParams.get('start') ?? undefined) : undefined;
+    return { title, item: [post, ...parseReplies($, link)], lastPageStart };
 }
 
 async function handler(ctx) {
@@ -74,7 +81,16 @@ async function handler(ctx) {
         throw new InvalidParameterError('Use a numeric topic ID and all or author for the reply scope.');
     }
     const link = `https://www.douban.com/group/topic/${id}/`;
-    const { data } = await got(link, { headers: { Cookie: config.douban.cookie }, searchParams: author === 'author' ? { author: 1 } : undefined });
+    const headers = { Cookie: config.douban.cookie };
+    const searchParams = author === 'author' ? { author: '1' } : undefined;
+    const { data } = await got(link, { headers, searchParams });
     const result = parseGroupTopic(data, link);
-    return { title: `${result.title} - 豆瓣小组帖子`, link, item: result.item };
+    let item = result.item;
+    if (result.lastPageStart) {
+        // Keep the post and replace the oldest replies with those on the last page
+        const pageUrl = `${link}?${new URLSearchParams({ ...searchParams, start: result.lastPageStart })}`;
+        const { data: lastPage } = await got(pageUrl, { headers });
+        item = [item[0], ...parseReplies(load(lastPage), link, pageUrl)];
+    }
+    return { title: `${result.title} - 豆瓣小组帖子`, link, item };
 }