From ff3d9a79becd5aadef9db050d0323317d261880a Mon Sep 17 00:00:00 2001 From: DIYgod Date: Sat, 9 May 2026 17:16:25 +0800 Subject: [PATCH] fix(route): update techflowpost routes --- lib/routes/techflowpost/express.ts | 39 ++-- lib/routes/techflowpost/featured.ts | 41 +--- lib/routes/techflowpost/index.ts | 38 ++-- lib/routes/techflowpost/techflowpost.test.ts | 156 +++++++++++++++ lib/routes/techflowpost/utils.ts | 192 +++++++++++++++++++ 5 files changed, 386 insertions(+), 80 deletions(-) create mode 100644 lib/routes/techflowpost/techflowpost.test.ts create mode 100644 lib/routes/techflowpost/utils.ts diff --git a/lib/routes/techflowpost/express.ts b/lib/routes/techflowpost/express.ts index 72f82fb46b..e3eda7cf93 100644 --- a/lib/routes/techflowpost/express.ts +++ b/lib/routes/techflowpost/express.ts @@ -1,10 +1,7 @@ -import dayjs from 'dayjs'; - import type { Route } from '@/types'; import { ViewType } from '@/types'; -import got from '@/utils/got'; -import { parseDate } from '@/utils/parse-date'; -import timezone from '@/utils/timezone'; + +import { getNewsflashItems, rootUrl } from './utils'; export const route: Route = { path: '/express', @@ -13,34 +10,26 @@ export const route: Route = { example: '/techflowpost/express', radar: [ { - source: ['techflowpost.com/newsletter/index.html'], + source: ['techflowpost.com/zh-CN/newsletter'], }, ], name: '快讯', + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, maintainers: ['nczitzk'], handler, - url: 'techflowpost.com/', + url: 'techflowpost.com/zh-CN/newsletter', }; async function handler(ctx) { - const rootUrl = 'https://www.techflowpost.com'; - const currentUrl = `${rootUrl}/newsletter/index.html`; - - const { data: response } = await got.post('https://www.techflowpost.com/ashx/newflash_index.ashx', { - form: { - pageindex: 1, - pagesize: ctx.req.query('limit') ?? 50, - time: dayjs().format('YYYY/M/D HH:mm:ss'), - }, - }); - - const items = response.content.map((item) => ({ - title: item.stitle, - link: `${rootUrl}/newsletter/detail_${item.nnewflash_id}.html`, - pubDate: timezone(parseDate(item.dcreate_time), +8), - updated: timezone(parseDate(item.dmodi_time), +8), - description: item.scontent, - })); + const currentUrl = `${rootUrl}/zh-CN/newsletter`; + const items = await getNewsflashItems(ctx.req.query('limit') ?? 50); return { title: '深潮TechFlow - 快讯', diff --git a/lib/routes/techflowpost/featured.ts b/lib/routes/techflowpost/featured.ts index 5c60913a22..bba3247b84 100644 --- a/lib/routes/techflowpost/featured.ts +++ b/lib/routes/techflowpost/featured.ts @@ -1,8 +1,7 @@ import type { Route } from '@/types'; import { ViewType } from '@/types'; -import got from '@/utils/got'; -import { parseDate } from '@/utils/parse-date'; -import timezone from '@/utils/timezone'; + +import { getArticleItems, rootUrl } from './utils'; export const route: Route = { path: '/featured/:category?', @@ -15,20 +14,20 @@ export const route: Route = { features: { requireConfig: false, requirePuppeteer: false, - antiCrawler: false, + antiCrawler: true, supportBT: false, supportPodcast: false, supportScihub: false, }, radar: [ { - source: ['techflowpost.com/article/index.html'], + source: ['techflowpost.com/zh-CN/article'], }, ], name: '精选', maintainers: ['zhenlohuang'], handler, - url: 'techflowpost.com/article/index.html', + url: 'techflowpost.com/zh-CN/article', description: `| 全部 | 行业 & 项目观察 | 项目简介 | 项目动态 | 赛道解读 | 播客笔记 | 交易观察 | VC 洞察 | 实用教程 | 人物故事 & 访谈 | 法律 & 监管动态 | 活动动态 | 交易所动态 | | ---- | --------------- | -------- | -------- | -------- | -------- | -------- | ------- | -------- | --------------- | --------------- | -------- | ---------- | | | 2040 | 2046 | 2047 | 2045 | 2044 | 2043 | 2042 | 2041 | 2039 | 2033 | 2032 | 2031 |`, @@ -52,34 +51,12 @@ const categoryMap: Record = { async function handler(ctx) { const category = ctx.req.param('category') ?? ''; const limit = ctx.req.query('limit') ?? 50; - - const rootUrl = 'https://www.techflowpost.com'; - const currentUrl = `${rootUrl}/article/index.html`; - - const formData: Record = { - pageindex: 1, - pagesize: limit, - is_specialnews: 'N', - }; - - if (category) { - formData.ncata_id = category; - } - - const { data: response } = await got.post('https://www.techflowpost.com/ashx/index.ashx', { - form: formData, + const currentUrl = `${rootUrl}/zh-CN/article`; + const items = await getArticleItems({ + category, + limit, }); - const items = response.content.map((item) => ({ - title: item.stitle, - author: item.sauthor_name, - link: `${rootUrl}/article/detail_${item.narticle_id}.html`, - category: [item.new_scata_name], - pubDate: timezone(parseDate(item.dcreate_time), +8), - updated: timezone(parseDate(item.dmodi_time), +8), - description: item.scontent, - })); - const categoryName = category && categoryMap[category] ? `（${categoryMap[category]}）` : ''; return { diff --git a/lib/routes/techflowpost/index.ts b/lib/routes/techflowpost/index.ts index 489a403103..cf57ce27e3 100644 --- a/lib/routes/techflowpost/index.ts +++ b/lib/routes/techflowpost/index.ts @@ -1,48 +1,40 @@ import type { Route } from '@/types'; import { ViewType } from '@/types'; -import got from '@/utils/got'; -import { parseDate } from '@/utils/parse-date'; -import timezone from '@/utils/timezone'; + +import { getArticleItems, rootUrl } from './utils'; export const route: Route = { path: '/', example: '/techflowpost', radar: [ { - source: ['techflowpost.com/'], + source: ['techflowpost.com/zh-CN'], }, ], name: '首页', categories: ['finance'], view: ViewType.Articles, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, maintainers: ['nczitzk'], handler, - url: 'techflowpost.com/', + url: 'techflowpost.com/zh-CN', }; async function handler(ctx) { - const rootUrl = 'https://www.techflowpost.com'; - - const { data: response } = await got.post('https://www.techflowpost.com/ashx/index.ashx', { - form: { - pageindex: 1, - pagesize: ctx.req.query('limit') ?? 50, - }, + const items = await getArticleItems({ + limit: ctx.req.query('limit') ?? 50, }); - const items = response.content.map((item) => ({ - title: item.stitle, - author: item.sauthor_name, - link: `${rootUrl}/article/detail_${item.narticle_id}.html`, - category: [item.new_scata_name], - pubDate: timezone(parseDate(item.dcreate_time), +8), - updated: timezone(parseDate(item.dmodi_time), +8), - description: item.scontent, - })); - return { title: '深潮TechFlow', - link: rootUrl, + link: `${rootUrl}/zh-CN`, item: items, }; } diff --git a/lib/routes/techflowpost/techflowpost.test.ts b/lib/routes/techflowpost/techflowpost.test.ts new file mode 100644 index 0000000000..3f674b176b --- /dev/null +++ b/lib/routes/techflowpost/techflowpost.test.ts @@ -0,0 +1,156 @@ +import { http, HttpResponse } from 'msw'; +import { describe, expect, it } from 'vitest'; + +import { route as expressRoute } from './express'; +import { route as featuredRoute } from './featured'; +import { route as indexRoute } from './index'; + +const challengeArg = '7FC0A515A247CA56A8EE791EF40FF1CAEB93AAA6'; +const expectedCookie = 'acw_sc__v2=69fef11a2a690097fa41a9b697a798108d4fb906'; + +const challenge = ``; + +function createCtx({ category, limit = '1' }: { category?: string; limit?: string } = {}) { + return { + req: { + param: (name: string) => (name === 'category' ? category : undefined), + query: (name: string) => (name === 'limit' ? limit : undefined), + }, + }; +} + +describe('/techflowpost', () => { + it('builds the article feed from the current client API after acw challenge', async () => { + const { default: server } = await import('@/setup.test'); + + server.use( + http.post('https://www.techflowpost.com/ashx/index.ashx', () => HttpResponse.json({ message: 'gone' }, { status: 410 })), + http.get('https://www.techflowpost.com/api/client/articles', ({ request }) => { + const url = new URL(request.url); + expect(url.searchParams.get('page')).toBe('1'); + expect(url.searchParams.get('page_size')).toBe('1'); + + if (!request.headers.get('cookie')?.includes(expectedCookie)) { + return HttpResponse.html(challenge); + } + + return HttpResponse.json({ + data: [ + { + id: 31495, + title: 'Article title', + abstract: 'Article abstract', + picture: '/upload/images/article.png', + author: { name: 'TechFlow' }, + category: { name: 'Trading' }, + labels: [{ label: 'AI' }], + created_at: '2026-05-09T06:40:11.209Z', + updated_at: '2026-05-09T08:31:01.272Z', + }, + ], + }); + }), + http.get('https://www.techflowpost.com/api/client/articles/31495', ({ request }) => { + expect(request.headers.get('cookie')).toContain(expectedCookie); + + return HttpResponse.json({ + article: { + content: '

Full article content

', + }, + }); + }) + ); + + const feed = await indexRoute.handler(createCtx()); + expect(feed.item).toHaveLength(1); + expect(feed.item[0].title).toBe('Article title'); + expect(feed.item[0].link).toBe('https://www.techflowpost.com/zh-CN/article/31495'); + expect(feed.item[0].description).toContain('Full article content'); + expect(feed.item[0].category).toEqual(['Trading', 'AI']); + }); + + it('passes category_id for featured article feeds', async () => { + const { default: server } = await import('@/setup.test'); + + server.use( + http.post('https://www.techflowpost.com/ashx/index.ashx', () => HttpResponse.json({ message: 'gone' }, { status: 410 })), + http.get('https://www.techflowpost.com/api/client/articles', ({ request }) => { + const url = new URL(request.url); + expect(url.searchParams.get('category_id')).toBe('2043'); + + if (!request.headers.get('cookie')?.includes(expectedCookie)) { + return HttpResponse.html(challenge); + } + + return HttpResponse.json({ + data: [ + { + id: 31496, + title: 'Featured article', + abstract: 'Featured abstract', + author: { name: 'TechFlow' }, + category: { name: 'Trading' }, + labels: [], + created_at: '2026-05-09T07:27:22.240Z', + updated_at: '2026-05-09T08:32:09.789Z', + }, + ], + }); + }), + http.get('https://www.techflowpost.com/api/client/articles/31496', () => + HttpResponse.json({ + article: { + content: '

Featured content

', + }, + }) + ) + ); + + const feed = await featuredRoute.handler(createCtx({ category: '2043' })); + expect(feed.item[0].title).toBe('Featured article'); + expect(feed.item[0].description).toContain('Featured content'); + }); + + it('builds the express feed from the current newsflash API', async () => { + const { default: server } = await import('@/setup.test'); + + server.use( + http.post('https://www.techflowpost.com/ashx/newflash_index.ashx', () => HttpResponse.json({ message: 'gone' }, { status: 410 })), + http.get('https://www.techflowpost.com/api/client/newsflashes', ({ request }) => { + const url = new URL(request.url); + expect(url.searchParams.get('page')).toBe('1'); + expect(url.searchParams.get('page_size')).toBe('1'); + + if (!request.headers.get('cookie')?.includes(expectedCookie)) { + return HttpResponse.html(challenge); + } + + return HttpResponse.json({ + data: [ + { + id: 122059, + title: 'Newsflash title', + abstract: 'Newsflash abstract', + url: 'https://example.com/source', + created_at: '2026-05-09T08:21:45.537Z', + updated_at: '2026-05-09T08:33:21.834Z', + }, + ], + }); + }), + http.get('https://www.techflowpost.com/api/client/newsflashes/122059', ({ request }) => { + expect(request.headers.get('cookie')).toContain(expectedCookie); + + return HttpResponse.json({ + content: '

Newsflash content

', + }); + }) + ); + + const feed = await expressRoute.handler(createCtx()); + expect(feed.item).toHaveLength(1); + expect(feed.item[0].title).toBe('Newsflash title'); + expect(feed.item[0].link).toBe('https://www.techflowpost.com/zh-CN/newsletter/122059'); + expect(feed.item[0].description).toContain('Newsflash content'); + }); +}); diff --git a/lib/routes/techflowpost/utils.ts b/lib/routes/techflowpost/utils.ts new file mode 100644 index 0000000000..acaabc2f3a --- /dev/null +++ b/lib/routes/techflowpost/utils.ts @@ -0,0 +1,192 @@ +import { config } from '@/config'; +import type { DataItem } from '@/types'; +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import { parseDate } from '@/utils/parse-date'; + +import { getAcwScV2ByArg1 } from '../5eplay/utils'; + +const rootUrl = 'https://www.techflowpost.com'; +const apiRootUrl = `${rootUrl}/api`; +const uploadRootUrl = 'https://upload.techflowpost.com'; +const locale = 'zh-CN'; + +let acwScV2Cookie = ''; + +type ApiListResponse = { + data?: T[]; +}; + +type Label = { + label?: string; +}; + +type Article = { + id: number; + title: string; + abstract?: string; + picture?: string; + author?: { + name?: string; + }; + category?: { + name?: string; + }; + labels?: Label[]; + created_at?: string; + updated_at?: string; +}; + +type ArticleDetailResponse = { + article?: { + content?: string; + }; +}; + +type Newsflash = { + id: number; + title: string; + abstract?: string; + url?: string; + created_at?: string; + updated_at?: string; +}; + +type NewsflashDetailResponse = { + content?: string; +}; + +function getHeaders(referer: string, cookie = acwScV2Cookie) { + return { + accept: 'application/json, text/plain, */*', + 'accept-language': locale, + referer, + 'user-agent': config.trueUA, + ...(cookie ? { cookie } : {}), + }; +} + +function getAcwScV2Cookie(response: string) { + const arg1 = response.match(/var arg1='(.*?)';/)?.[1]; + if (!arg1) { + return ''; + } + return `acw_sc__v2=${getAcwScV2ByArg1(arg1)}`; +} + +async function requestApi(endpoint: string, referer: string, searchParams?: Record) { + const request = () => + got.get(`${apiRootUrl}${endpoint}`, { + searchParams, + headers: getHeaders(referer), + }); + + const { data } = await request(); + if (typeof data !== 'string') { + return data as T; + } + + const cookie = getAcwScV2Cookie(data); + if (!cookie) { + throw new Error('TechFlow API returned an unexpected non-JSON response.'); + } + + acwScV2Cookie = cookie; + const retryResponse = await request(); + if (typeof retryResponse.data === 'string') { + throw new TypeError('TechFlow API still returned an anti-crawler challenge after retry.'); + } + + return retryResponse.data as T; +} + +function getPictureUrl(picture?: string) { + if (!picture) { + return; + } + return new URL(picture, uploadRootUrl).href; +} + +function getCategories(article: Article) { + return [...new Set([article.category?.name, ...(article.labels?.map((label) => label.label) ?? [])].filter(Boolean))] as string[]; +} + +function getArticleItem(article: Article, content?: string): DataItem { + const link = `${rootUrl}/${locale}/article/${article.id}`; + const categories = getCategories(article); + const image = getPictureUrl(article.picture); + + return { + title: article.title, + author: article.author?.name, + link, + category: categories.length ? categories : undefined, + pubDate: article.created_at ? parseDate(article.created_at) : undefined, + updated: article.updated_at ? parseDate(article.updated_at) : undefined, + description: content || article.abstract, + image, + }; +} + +function getNewsflashItem(newsflash: Newsflash, content?: string): DataItem { + return { + title: newsflash.title, + link: `${rootUrl}/${locale}/newsletter/${newsflash.id}`, + pubDate: newsflash.created_at ? parseDate(newsflash.created_at) : undefined, + updated: newsflash.updated_at ? parseDate(newsflash.updated_at) : undefined, + description: content || newsflash.abstract, + }; +} + +async function getArticleItems({ category, limit }: { category?: string; limit: string | number }) { + const link = `${rootUrl}/${locale}/article`; + const searchParams: Record = { + page: 1, + page_size: limit, + }; + + if (category) { + searchParams.category_id = category; + } + + const response = await requestApi>('/client/articles', link, searchParams); + if (!Array.isArray(response.data)) { + throw new TypeError('TechFlow API returned an invalid article list.'); + } + + return await Promise.all( + response.data.map((article) => + cache.tryGet(`techflowpost:article:${article.id}`, async () => { + const itemLink = `${rootUrl}/${locale}/article/${article.id}`; + const detail = await requestApi(`/client/articles/${article.id}`, itemLink); + + return getArticleItem(article, detail.article?.content); + }) + ) + ); +} + +async function getNewsflashItems(limit: string | number) { + const link = `${rootUrl}/${locale}/newsletter`; + const response = await requestApi>('/client/newsflashes', link, { + page: 1, + page_size: limit, + }); + + if (!Array.isArray(response.data)) { + throw new TypeError('TechFlow API returned an invalid newsflash list.'); + } + + return await Promise.all( + response.data.map((newsflash) => + cache.tryGet(`techflowpost:newsflash:${newsflash.id}`, async () => { + const itemLink = `${rootUrl}/${locale}/newsletter/${newsflash.id}`; + const detail = await requestApi(`/client/newsflashes/${newsflash.id}`, itemLink); + + return getNewsflashItem(newsflash, detail.content); + }) + ) + ); +} + +export { getArticleItems, getNewsflashItems, rootUrl };