feat(route/chikubi): refactor URL fetching and add new routes (#16758)

* refactor(route/chikubi): switch to native /feed for article URL fetching

* feat: add new routes

* refactor: remove decodeURIComponent

* fix: Handle unescaped characters in tag URLs

* chore: use '@/utils/rss-parser' instead of 'rss-parser'

* chore: update description

* feat(utils): add getPostById and getPostsByIdList for REST API content fetching

* refactor(search): use getPostsByIdList instead of processItems

* fix: Handle unescaped characters

* refactor: Separate index and navigation

* feat: optimize post fetching by id and add category & tag functions

* refactor: use api to get posts for tags and categories

* chore: directly use url

* fix: use API to fetch 'best'

* fix: destructure post object in map function

* feat: optimize getPosts function for reusability

* refactor: remove wrapper functions for getBySlug and getPostsBy

lib/routes/chikubi/category.ts

@@ -0,0 +1,41 @@
+import { Route, Data } from '@/types';
+import { getBySlug, getPostsBy } from './utils';
+
+export const route: Route = {
+    path: '/category/:keyword',
+    categories: ['multimedia'],
+    example: '/chikubi/category/nipple-lesbian',
+    parameters: { keyword: 'Keyword' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Category',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: 'Category',
+            source: ['chikubi.jp/category/:keyword'],
+            target: '/category/:keyword',
+        },
+    ],
+};
+
+async function handler(ctx): Promise<Data> {
+    const baseUrl = 'https://chikubi.jp';
+    const { keyword } = ctx.req.param();
+    const { id, name } = await getBySlug('category', keyword);
+
+    const items = await getPostsBy('category', id);
+
+    return {
+        title: `Category: ${name} - chikubi.jp`,
+        link: `${baseUrl}/category/${keyword}`,
+        item: items,
+    };
+}

lib/routes/chikubi/index.ts

@@ -1,13 +1,10 @@
 import { Route, Data } from '@/types';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import { processItems } from './utils';
+import { getPosts } from './utils';
 
 export const route: Route = {
-    path: '/:category?',
+    path: '/',
     categories: ['multimedia'],
     example: '/chikubi',
-    parameters: { category: '分類，見下表，默認爲最新' },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -16,60 +13,54 @@ export const route: Route = {
         supportPodcast: false,
         supportScihub: false,
     },
-    name: 'Category',
-    maintainers: ['snowagar25'],
+    name: '最新記事',
+    maintainers: ['SnowAgar25'],
     handler,
-    description: `| 最新 | 殿堂 | 動畫 | VR | 漫畫 | 音聲 | CG |
-  | ------ | ---- | ----- | -- | ----- | ----- | -- |
-  | (empty) | best | video | vr | comic | voice | cg |`,
     radar: [
         {
-            source: ['chikubi.jp/:category', 'chikubi.jp/'],
-            target: '/:category',
+            title: '最新記事',
+            source: ['chikubi.jp/'],
+            target: '/',
+        },
+        {
+            title: '殿堂',
+            source: ['chikubi.jp/best-nipple-article'],
+            target: '/best',
+        },
+        {
+            title: '動畫',
+            source: ['chikubi.jp/nipple-video'],
+            target: '/video',
+        },
+        {
+            title: 'VR',
+            source: ['chikubi.jp/nipple-video-category/cat-nipple-video-vr'],
+            target: '/vr',
+        },
+        {
+            title: '漫畫',
+            source: ['chikubi.jp/comic'],
+            target: '/comic',
+        },
+        {
+            title: '音聲',
+            source: ['chikubi.jp/voice'],
+            target: '/voice',
+        },
+        {
+            title: 'CG・イラスト',
+            source: ['chikubi.jp/cg'],
+            target: '/cg',
         },
     ],
 };
 
-const categoryMap = {
-    '': { url: '/page/1', title: '最新', selector: '.article_list_area > article > a' },
-    best: { url: '/best-nipple-article', title: '殿堂', selector: '.article-list:first .title > a' },
-    video: { url: '/nipple-video', title: '動畫', selector: 'ul.video_list > li > a' },
-    vr: { url: '/nipple-video-category/cat-nipple-video-vr', title: 'VR', selector: 'ul.video_list > li > a' },
-    comic: { url: '/comic', title: '漫畫', selector: '.section:nth-of-type(2) .list-doujin .photo a' },
-    voice: { url: '/voice', title: '音聲', selector: 'ul.list-doujin > li > .photo > a' },
-    cg: { url: '/cg', title: 'CG', selector: 'ul.list-doujin > li > .photo > a' },
-};
-
-async function handler(ctx): Promise<Data> {
-    const category = ctx.req.param('category') ?? '';
-    const baseUrl = 'https://chikubi.jp';
-
-    const { url, title, selector } = categoryMap[category];
-
-    const response = await got(`${baseUrl}${url}`);
-    const $ = load(response.data);
-
-    let list = $(selector)
-        .toArray()
-        .map((item) => {
-            const $item = $(item);
-            return {
-                title: $item.text().trim(),
-                link: new URL($item.attr('href') ?? '', baseUrl).href,
-            };
-        });
-
-    // 限制殿堂最多獲取30個
-    if (category === 'best') {
-        list = list.slice(0, 30);
-    }
-
-    // 獲取內文
-    const items = await processItems(list);
+async function handler(): Promise<Data> {
+    const items = await getPosts();
 
     return {
-        title: `${title} - chikubi.jp`,
-        link: `${baseUrl}${url}`,
+        title: '最新記事 - chikubi.jp',
+        link: 'https://chikubi.jp',
         item: items,
     };
 }

lib/routes/chikubi/namespace.ts

@@ -3,4 +3,13 @@ import type { Namespace } from '@/types';
 export const namespace: Namespace = {
     name: '乳首ふぇち',
     url: 'chikubi.jp',
+    description: `:::tip
+The content of 乳首ふぇち is divided into two parts:
+
+Works: Only reposts official product descriptions.
+Posts: Contains the website author's thoughts and additional information.
+
+Sometimes a product may exist in both posts and works.
+Sometimes there might be only a single post without any reposted work, and vice versa.
+:::`,
 };

lib/routes/chikubi/navigation.ts

@@ -0,0 +1,66 @@
+import { Route, Data } from '@/types';
+import { getBySlug, getPostsBy, processItems } from './utils';
+import parser from '@/utils/rss-parser';
+
+export const route: Route = {
+    path: '/:keyword',
+    categories: ['multimedia'],
+    example: '/chikubi',
+    parameters: { keyword: '導覽列，見下表，默認爲最新' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Navigation',
+    maintainers: ['SnowAgar25'],
+    handler,
+    description: `| 殿堂 | 動畫 | VR | 漫畫 | 音聲 | CG・イラスト |
+  | ---- | ----- | -- | ----- | ----- | -- |
+  | best | video | vr | comic | voice | cg |`,
+};
+
+const navigationItems = {
+    video: { url: '/nipple-video', title: '動畫' },
+    vr: { url: '/nipple-video-category/cat-nipple-video-vr', title: 'VR' },
+    comic: { url: '/comic', title: '漫畫' },
+    voice: { url: '/voice', title: '音聲' },
+    cg: { url: '/cg', title: 'CG' },
+};
+
+async function handler(ctx): Promise<Data> {
+    const keyword = ctx.req.param('keyword') ?? '';
+    const baseUrl = 'https://chikubi.jp';
+
+    if (keyword === 'best') {
+        const { id } = await getBySlug('category', 'nipple-best');
+        const items = await getPostsBy('category', id);
+
+        return {
+            title: '殿堂 - chikubi.jp',
+            link: `${baseUrl}/best-nipple-article`,
+            item: items,
+        };
+    } else {
+        const { url, title } = navigationItems[keyword];
+
+        const feed = await parser.parseURL(`${baseUrl}${url}/feed`);
+
+        const list = feed.items.map((item) => ({
+            title: item.title,
+            link: item.link,
+        }));
+
+        // 獲取內文
+        const items = await processItems(list);
+
+        return {
+            title: `${title} - chikubi.jp`,
+            link: `${baseUrl}${url}`,
+            item: items,
+        };
+    }
+}

lib/routes/chikubi/nipple-video-category.ts

@@ -0,0 +1,49 @@
+import { Route, Data } from '@/types';
+import { processItems } from './utils';
+import parser from '@/utils/rss-parser';
+
+export const route: Route = {
+    path: '/nipple-video-category/:keyword',
+    categories: ['multimedia'],
+    example: '/chikubi/nipple-video-category/cat-nipple-video-god',
+    parameters: { keyword: 'Keyword' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '動画カテゴリー',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: '動画カテゴリー',
+            source: ['chikubi.jp/nipple-video-category/:keyword'],
+            target: '/nipple-video-category/:keyword',
+        },
+    ],
+};
+
+async function handler(ctx): Promise<Data> {
+    const { keyword } = ctx.req.param();
+    const baseUrl = 'https://chikubi.jp';
+    const url = `/nipple-video-category/${encodeURIComponent(keyword)}`;
+
+    const feed = await parser.parseURL(`${baseUrl}${url}/feed`);
+
+    const list = feed.items.map((item) => ({
+        title: item.title,
+        link: item.link,
+    }));
+
+    const items = await processItems(list);
+
+    return {
+        title: `動画カテゴリー: ${feed.title?.split('-')[0]} - chikubi.jp`,
+        link: `${baseUrl}${url}`,
+        item: items,
+    };
+}

lib/routes/chikubi/nipple-video-maker.ts

@@ -0,0 +1,49 @@
+import { Route, Data } from '@/types';
+import { processItems } from './utils';
+import parser from '@/utils/rss-parser';
+
+export const route: Route = {
+    path: '/nipple-video-maker/:keyword',
+    categories: ['multimedia'],
+    example: '/chikubi/nipple-video-maker/nipple-video-maker-nh',
+    parameters: { keyword: 'Keyword' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'AVメーカー',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: 'AVメーカー',
+            source: ['chikubi.jp/nipple-video-maker/:keyword'],
+            target: '/nipple-video-maker/:keyword',
+        },
+    ],
+};
+
+async function handler(ctx): Promise<Data> {
+    const { keyword } = ctx.req.param();
+    const baseUrl = 'https://chikubi.jp';
+    const url = `/nipple-video-maker/${encodeURIComponent(keyword)}`;
+
+    const feed = await parser.parseURL(`${baseUrl}${url}/feed`);
+
+    const list = feed.items.map((item) => ({
+        title: item.title,
+        link: item.link,
+    }));
+
+    const items = await processItems(list);
+
+    return {
+        title: `AVメーカー: ${feed.title?.split('-')[0]} - chikubi.jp`,
+        link: `${baseUrl}${url}`,
+        item: items,
+    };
+}

lib/routes/chikubi/search.ts

@@ -0,0 +1,39 @@
+import { Route, Data } from '@/types';
+import { getPosts } from './utils';
+import got from '@/utils/got';
+
+export const route: Route = {
+    path: '/search/:keyword',
+    categories: ['multimedia'],
+    example: '/chikubi/search/ギャップ',
+    parameters: { keyword: 'Keyword' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Search',
+    maintainers: ['SnowAgar25'],
+    handler,
+};
+
+async function handler(ctx): Promise<Data> {
+    const { keyword } = ctx.req.param();
+    const baseUrl = 'https://chikubi.jp';
+    const searchUrl = `${baseUrl}/wp-json/wp/v2/search?search=${keyword}`;
+
+    const response = await got.get(searchUrl);
+    const searchResults = response.data;
+
+    const postIds = searchResults.map((item) => item.id.toString());
+    const items = await getPosts(postIds);
+
+    return {
+        title: `Search: ${keyword} - chikubi.jp`,
+        link: `${baseUrl}/search/${encodeURIComponent(keyword)}`,
+        item: items,
+    };
+}

lib/routes/chikubi/tag.ts

@@ -0,0 +1,41 @@
+import { Route, Data } from '@/types';
+import { getBySlug, getPostsBy } from './utils';
+
+export const route: Route = {
+    path: '/tag/:keyword',
+    categories: ['multimedia'],
+    example: '/chikubi/tag/ドリームチケット',
+    parameters: { keyword: 'Keyword' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Tag',
+    maintainers: ['SnowAgar25'],
+    handler,
+    radar: [
+        {
+            title: 'Tag',
+            source: ['chikubi.jp/tag/:keyword'],
+            target: '/tag/:keyword',
+        },
+    ],
+};
+
+async function handler(ctx): Promise<Data> {
+    const baseUrl = 'https://chikubi.jp';
+    const { keyword } = ctx.req.param();
+    const { id, name } = await getBySlug('tag', keyword);
+
+    const items = await getPostsBy('tag', id);
+
+    return {
+        title: `Tag: ${name} - chikubi.jp`,
+        link: `${baseUrl}/category/${keyword}`,
+        item: items,
+    };
+}

lib/routes/chikubi/utils.ts

@@ -4,17 +4,7 @@ import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
 
-interface ListItem {
-    title: string;
-    link: string;
-}
-
-interface ContentSelectors {
-    title: string;
-    description: string[];
-}
-
-const contentTypes: Record<string, ContentSelectors> = {
+const CONTENT_TYPES = {
     doujin: {
         title: '.doujin-title',
         description: ['.doujin-detail', '.section', '.area-buy > a.btn'],
@@ -29,7 +19,7 @@ const contentTypes: Record<string, ContentSelectors> = {
     },
 };
 
-function getContentType(link: string): keyof typeof contentTypes {
+function getContentType(link: string): keyof typeof CONTENT_TYPES {
     const typePatterns = {
         doujin: ['/cg/', '/comic/', '/voice/'],
         video: ['/nipple-video/'],
@@ -38,14 +28,14 @@ function getContentType(link: string): keyof typeof contentTypes {
 
     for (const [type, patterns] of Object.entries(typePatterns)) {
         if (patterns.some((pattern) => link.includes(pattern))) {
-            return type as keyof typeof contentTypes;
+            return type as keyof typeof CONTENT_TYPES;
         }
     }
 
     throw new Error(`Unknown content type for link: ${link}`);
 }
 
-export async function processItems(list: ListItem[]): Promise<DataItem[]> {
+export async function processItems(list): Promise<DataItem[]> {
     const items = await Promise.all(
         list.map((item) =>
             cache.tryGet(item.link, async () => {
@@ -53,17 +43,10 @@ export async function processItems(list: ListItem[]): Promise<DataItem[]> {
                 const $ = load(detailResponse.data);
 
                 const contentType = getContentType(item.link);
-                const selectors = contentTypes[contentType];
+                const selectors = CONTENT_TYPES[contentType];
 
                 const title = $(selectors.title).text().trim() || item.title;
-                const description = selectors.description
-                    .map((selector) =>
-                        $(selector)
-                            .map((_, el) => $(el).clone().wrap('<div>').parent().html())
-                            .toArray()
-                            .join('')
-                    )
-                    .join('');
+                const description = processDescription(selectors.description.map((selector) => $(selector).prop('outerHTML')).join(''));
 
                 const pubDateStr = $('meta[property="article:published_time"]').attr('content');
                 const pubDate = pubDateStr ? parseDate(pubDateStr) : undefined;
@@ -80,3 +63,73 @@ export async function processItems(list: ListItem[]): Promise<DataItem[]> {
 
     return items.filter((item): item is DataItem => item !== null);
 }
+
+function processDescription(description: string): string {
+    const $ = load(description);
+    return $('body')
+        .children()
+        .map((_, el) => $(el).clone().wrap('<div>').parent().html())
+        .toArray()
+        .join('');
+}
+
+const WP_REST_API_URL = 'https://chikubi.jp/wp-json/wp/v2';
+
+export async function getPosts(ids?: string[]): Promise<DataItem[]> {
+    const url = `${WP_REST_API_URL}/posts${ids?.length ? `?include=${ids.join(',')}` : ''}`;
+
+    const cachedData = await cache.tryGet(url, async () => {
+        const response = await got(url);
+        const data = JSON.parse(response.body);
+
+        if (!Array.isArray(data)) {
+            throw new TypeError('No posts found for the given IDs');
+        }
+
+        return data.map(({ title, link, date, content }) => ({
+            title: title.rendered,
+            link,
+            pubDate: parseDate(date),
+            description: processDescription(content.rendered),
+        }));
+    });
+
+    return (Array.isArray(cachedData) ? cachedData : []).filter((item): item is DataItem => item !== null);
+}
+
+const API_TYPES = {
+    tag: 'tags',
+    category: 'categories',
+};
+
+export async function getBySlug<T extends keyof typeof API_TYPES>(type: T, slug: string): Promise<{ id: number; name: string }> {
+    const url = `${WP_REST_API_URL}/${API_TYPES[type]}?slug=${encodeURIComponent(slug)}`;
+    const { body } = await got(url);
+    const data = JSON.parse(body);
+
+    if (data?.[0]) {
+        const { id, name } = data[0];
+        return { id, name };
+    }
+    throw new Error(`No ${type} found for slug: ${slug}`);
+}
+
+export async function getPostsBy<T extends keyof typeof API_TYPES>(type: T, id: number): Promise<DataItem[]> {
+    const url = `${WP_REST_API_URL}/posts?${API_TYPES[type]}=${id}`;
+    const cachedData = await cache.tryGet(url, async () => {
+        const { body } = await got(url);
+        const data = JSON.parse(body);
+
+        if (Array.isArray(data) && data.length > 0) {
+            return data.map(({ title, link, date, content }) => ({
+                title: title.rendered,
+                link,
+                pubDate: parseDate(date),
+                description: processDescription(content.rendered),
+            }));
+        }
+        return [];
+    });
+
+    return (Array.isArray(cachedData) ? cachedData : []).filter((item): item is DataItem => item !== null);
+}