diff --git a/lib/routes/threads/index.ts b/lib/routes/threads/index.ts index ccbb76255d..c1d739b253 100644 --- a/lib/routes/threads/index.ts +++ b/lib/routes/threads/index.ts @@ -1,12 +1,11 @@ -import { JSDOM } from 'jsdom'; -import { JSONPath } from 'jsonpath-plus'; +import { load } from 'cheerio'; import type { Route } from '@/types'; import { ViewType } from '@/types'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -import { buildContent, extractTokens, getUserId, profileUrl, threadUrl } from './utils'; +import { buildContent, extractThreadItems, parseRouteOptions, profileUrl, threadUrl } from './utils'; export const route: Route = { path: '/:user/:routeParams?', @@ -37,103 +36,36 @@ Specify options (in the format of query string) in parameter \`routeParams\` to async function handler(ctx) { const { user, routeParams } = ctx.req.param(); - const { lsd } = await extractTokens(user); - const userId = await getUserId(user); + const options = parseRouteOptions(new URLSearchParams(routeParams)); - const params = new URLSearchParams(routeParams); - const debugJson: any = { - params: routeParams, - lsd, - }; + const response = await ofetch(profileUrl(user)); + const $ = load(response); - const options = { - showAuthorInTitle: params.get('showAuthorInTitle') ?? true, - showAuthorInDesc: params.get('showAuthorInDesc') ?? true, - showAuthorAvatarInDesc: params.get('showAuthorAvatarInDesc') ?? false, - showQuotedInTitle: params.get('showQuotedInTitle') ?? true, - showQuotedAuthorAvatarInDesc: params.get('showQuotedAuthorAvatarInDesc') ?? false, - showEmojiForQuotesAndReply: params.get('showEmojiForQuotesAndReply') ?? true, - replies: params.get('replies') ?? false, - }; + const threadsData = extractThreadItems($); - const response = await ofetch(profileUrl(user), { - headers: { - 'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'Accept-Encoding': 'gzip, br', - 'Accept-Language': 'zh-CN,zh;q=0.9', - 'Cache-Control': 'no-cache', - Pragma: 'no-cache', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - 'Upgrade-Insecure-Requests': '1', - }, - }); - - const dom = new JSDOM(response); - const { document } = dom.window; - - let threadsData: ThreadItem[] | null = null; - for (const el of document.querySelectorAll('script[data-sjs]')) { - try { - const data = JSONPath({ - path: '$..thread_items[0]', - json: JSON.parse(el.textContent || ''), - }); - - if (data?.length > 0) { - threadsData = data; - break; - } - } catch { - // Skip invalid JSON - } - } - - if (!threadsData) { + if (!threadsData.length) { throw new Error('Failed to fetch thread data'); } - debugJson.profileId = userId; - debugJson.response = { response: threadsData }; - - const userData: ThreadUser = threadsData[0]?.post?.user || { username: user, profile_pic_url: '' }; + ctx.set('json', threadsData); const items = threadsData .filter((item) => user === item.post.user?.username) - .map((item) => ({ - author: user, - title: buildContent(item, options).title, - description: buildContent(item, options).description, - pubDate: parseDate(item.post.taken_at, 'X'), - link: threadUrl(item.post.code), - })); - - debugJson.items = items; - ctx.set('json', debugJson); + .map((item) => { + const { title, description } = buildContent(item, options); + return { + author: user, + title, + description, + pubDate: parseDate(item.post.taken_at, 'X'), + link: threadUrl(item.post.code), + }; + }); return { title: `${user} (@${user}) on Threads`, link: profileUrl(user), - image: userData?.profile_pic_url, + image: threadsData[0].post.user?.profile_pic_url, item: items, }; } - -interface ThreadUser { - username: string; - profile_pic_url: string; -} - -interface ThreadItem { - post: { - user?: ThreadUser; - taken_at: number; - code: string; - caption?: { - text: string; - }; - }; -} diff --git a/lib/routes/threads/search.ts b/lib/routes/threads/search.ts new file mode 100644 index 0000000000..a802675cb2 --- /dev/null +++ b/lib/routes/threads/search.ts @@ -0,0 +1,64 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import { ViewType } from '@/types'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +import { buildContent, extractThreadItems, parseRouteOptions, threadUrl } from './utils'; + +export const route: Route = { + path: '/search/:keyword/:routeParams?', + categories: ['social-media'], + view: ViewType.SocialMedia, + example: '/threads/search/RSS', + parameters: { + keyword: 'Search keyword', + routeParams: { + description: `Extra parameters, in the format of query string. Accepts the same options as User timeline, plus: + +| Key | Description | Accepts | Defaults to | +| ----------- | ----------- | -------------------------- | ----------- | +| \`serpType\` | Search type | \`tags\`/\`default\`/\`recent\` | \`tags\` |`, + }, + }, + name: 'Search', + maintainers: ['TonyRL'], + handler, +}; + +async function handler(ctx) { + const { keyword, routeParams } = ctx.req.param(); + const params = new URLSearchParams(routeParams); + const options = parseRouteOptions(params); + const serpType = params.get('serpType') ?? 'tags'; + + const link = `https://www.threads.com/search?q=${encodeURIComponent(keyword)}&serp_type=${serpType}`; + const response = await ofetch(link); + const $ = load(response); + + const threadsData = extractThreadItems($); + + if (!threadsData.length) { + throw new Error('Failed to fetch thread data'); + } + + ctx.set('json', threadsData); + + const items = threadsData.map((item) => { + const { title, description } = buildContent(item, options); + return { + author: item.post.user?.username, + title, + description, + pubDate: parseDate(item.post.taken_at, 'X'), + link: threadUrl(item.post.code), + }; + }); + + return { + title: `${keyword} - Search on Threads`, + link, + item: items, + }; +} diff --git a/lib/routes/threads/utils.ts b/lib/routes/threads/utils.ts index 70cd57f69d..b119ff2607 100644 --- a/lib/routes/threads/utils.ts +++ b/lib/routes/threads/utils.ts @@ -1,91 +1,56 @@ -import { load } from 'cheerio'; +import type { CheerioAPI } from 'cheerio'; import dayjs from 'dayjs'; -import { JSDOM } from 'jsdom'; -import { JSONPath } from 'jsonpath-plus'; -import NotFoundError from '@/errors/types/not-found'; -import cache from '@/utils/cache'; -import ofetch from '@/utils/ofetch'; +import { queryToBoolean } from '@/utils/readable-social'; -const profileUrl = (user: string) => `https://www.threads.com/@${user}`; -const threadUrl = (code: string) => `https://www.threads.com/t/${code}`; +export const profileUrl = (user: string) => `https://www.threads.com/@${user}`; +export const threadUrl = (code: string) => `https://www.threads.com/t/${code}`; -const USER_AGENT = 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1'; +export interface ThreadItem { + post: { + user?: { + username: string; + profile_pic_url: string; + }; + taken_at: number; + code: string; + caption?: { + text: string; + }; + }; +} -const extractTokens = async (user): Promise<{ lsd: string }> => { - const response = await ofetch(profileUrl(user), { - headers: { - 'User-Agent': USER_AGENT, - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'Accept-Encoding': 'gzip, br', - 'Accept-Language': 'zh-CN,zh;q=0.9', - 'Cache-Control': 'no-cache', - Pragma: 'no-cache', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - 'Upgrade-Insecure-Requests': '1', - }, - }); - - const $ = load(response); - const data = $('script:contains("LSD"):first').text(); - const lsd = data.match(/"LSD",\[\],\{"token":"([\w@-]+)"\},/)?.[1]; - - if (!lsd) { - throw new NotFoundError('LSD token not found'); - } - - return { lsd }; -}; - -const getUserId = async (user: string): Promise => { - const result = await cache.tryGet(`threads:userId:${user}`, async () => { - const response = await ofetch(profileUrl(user), { - headers: { - 'User-Agent': USER_AGENT, - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', - 'Accept-Encoding': 'gzip, br', - 'Accept-Language': 'zh-CN,zh;q=0.9', - 'Cache-Control': 'no-cache', - Pragma: 'no-cache', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - 'Upgrade-Insecure-Requests': '1', - }, - }); - - const dom = new JSDOM(response); - const { document } = dom.window; - - for (const el of document.querySelectorAll('script[data-sjs]')) { - try { - // the Threads payload types `user_id` as either a numeric or a string id - const data = JSONPath>({ - path: '$..user_id', - json: JSON.parse(el.textContent || ''), - }); - - if (data?.[0]) { - return String(data[0]); - } - } catch { - // Skip invalid JSON - } +const findThreadItems = (node, acc: ThreadItem[] = []): ThreadItem[] => { + if (node instanceof Object) { + if (Array.isArray(node.thread_items)) { + acc.push(...node.thread_items); + } + for (const value of Object.values(node)) { + findThreadItems(value, acc); } - - throw new NotFoundError('User ID not found'); - }); - - if (!result) { - throw new TypeError('Invalid user ID type'); } - return result; + return acc; }; +export const extractThreadItems = ($: CheerioAPI): ThreadItem[] => { + let threadsData: ThreadItem[] = []; + $('script[data-sjs]:contains("thread_items")').each((_, script) => { + threadsData = findThreadItems(JSON.parse($(script).text())); + return threadsData.length === 0; + }); + return threadsData; +}; + +export const parseRouteOptions = (params: URLSearchParams) => ({ + showAuthorInTitle: queryToBoolean(params.get('showAuthorInTitle')) ?? true, + showAuthorInDesc: queryToBoolean(params.get('showAuthorInDesc')) ?? true, + showAuthorAvatarInDesc: queryToBoolean(params.get('showAuthorAvatarInDesc')) ?? false, + showQuotedInTitle: queryToBoolean(params.get('showQuotedInTitle')) ?? true, + showQuotedAuthorAvatarInDesc: queryToBoolean(params.get('showQuotedAuthorAvatarInDesc')) ?? false, + showEmojiForQuotesAndReply: queryToBoolean(params.get('showEmojiForQuotesAndReply')) ?? true, + replies: queryToBoolean(params.get('replies')) ?? false, +}); + const hasMedia = (post) => post.image_versions2 || post.carousel_media || post.video_versions; const buildMedia = (post) => { @@ -108,7 +73,7 @@ const buildMedia = (post) => { return html; }; -const buildContent = (item, options) => { +export const buildContent = (item, options) => { let title = ''; let description = ''; const quotedPost = item.post.text_post_app_info?.share_info?.quoted_post; @@ -164,5 +129,3 @@ const buildContent = (item, options) => { } return { title, description }; }; - -export { buildContent, extractTokens, getUserId, profileUrl, threadUrl }; diff --git a/package.json b/package.json index d7c8ba7d20..6b746e2133 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,6 @@ "ip-regex": "5.0.0", "jsdom": "30.0.1", "json-bigint": "1.0.0", - "jsonpath-plus": "10.4.0", "jsrsasign": "11.1.5", "lru-cache": "11.5.2", "lz-string": "1.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83927f6423..0b55930ade 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -163,9 +163,6 @@ importers: json-bigint: specifier: 1.0.0 version: 1.0.0 - jsonpath-plus: - specifier: 10.4.0 - version: 10.4.0 jsrsasign: specifier: 11.1.5 version: 11.1.5 @@ -1605,18 +1602,6 @@ packages: '@jridgewell/trace-mapping@0.3.9': resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - '@jsep-plugin/assignment@1.3.0': - resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==} - engines: {node: '>= 10.16.0'} - peerDependencies: - jsep: ^0.4.0||^1.0.0 - - '@jsep-plugin/regex@1.0.4': - resolution: {integrity: sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg==} - engines: {node: '>= 10.16.0'} - peerDependencies: - jsep: ^0.4.0||^1.0.0 - '@keyv/serialize@1.1.1': resolution: {integrity: sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==} @@ -4653,10 +4638,6 @@ packages: canvas: optional: true - jsep@1.4.0: - resolution: {integrity: sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==} - engines: {node: '>= 10.16.0'} - jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} @@ -4689,11 +4670,6 @@ packages: jsonfile@6.2.1: resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} - jsonpath-plus@10.4.0: - resolution: {integrity: sha512-T92WWatJXmhBbKsgH/0hl+jxjdXrifi5IKeMY02DWggRxX0UElcbVzPlmgLTbvsPeW1PasQ6xE2Q75stkhGbsA==} - engines: {node: '>=18.0.0'} - hasBin: true - jsprim@1.4.2: resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} engines: {node: '>=0.6.0'} @@ -7435,14 +7411,6 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 - '@jsep-plugin/assignment@1.3.0(jsep@1.4.0)': - dependencies: - jsep: 1.4.0 - - '@jsep-plugin/regex@1.0.4(jsep@1.4.0)': - dependencies: - jsep: 1.4.0 - '@keyv/serialize@1.1.1': {} '@lifeomic/attempt@3.1.0': {} @@ -10212,8 +10180,6 @@ snapshots: transitivePeerDependencies: - '@noble/hashes' - jsep@1.4.0: {} - jsesc@3.1.0: {} json-bigint@1.0.0: @@ -10240,12 +10206,6 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 - jsonpath-plus@10.4.0: - dependencies: - '@jsep-plugin/assignment': 1.3.0(jsep@1.4.0) - '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) - jsep: 1.4.0 - jsprim@1.4.2: dependencies: assert-plus: 1.0.0