From 9787a92c843a5a8e398dd98622841ab1db25fd80 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Thu, 29 Feb 2024 18:02:51 +0800 Subject: [PATCH] feat: v3 migration - 14/n --- lib/v2/bjx/huanbao.js | 4 ++-- lib/v2/dlnews/category.js | 4 ++-- lib/v2/douban/people/status.js | 16 ++++++++-------- lib/v2/eleduck/posts.js | 4 ++-- lib/v2/gov/moa/moa.js | 8 ++++---- lib/v2/gov/pbc/zcyj.js | 2 +- lib/v2/gov/sichuan/deyang/govpublicinfo.js | 4 ++-- lib/v2/gov/sichuan/deyang/mztoday.js | 4 ++-- lib/v2/guokr/channel.js | 4 ++-- lib/v2/hoyolab/news.js | 4 ++-- lib/v2/kuaidi100/index.js | 4 ++-- lib/v2/kuaidi100/supported-company.js | 2 +- lib/v2/kuaidi100/utils.js | 22 +++++++++++----------- lib/v2/liveuamap/index.js | 8 ++------ lib/v2/newrank/utils.js | 10 +++++----- lib/v2/phoronix/index.js | 16 ++++++++-------- lib/v2/sensortower/blog.js | 2 +- lib/v2/udn/breaking-news.js | 4 ++-- lib/v2/wechat/tgchannel.js | 2 +- lib/v2/xiaoyuzhou/pickup.js | 4 ++-- 20 files changed, 62 insertions(+), 66 deletions(-) diff --git a/lib/v2/bjx/huanbao.js b/lib/v2/bjx/huanbao.js index 2f98821c09..68c83c6659 100644 --- a/lib/v2/bjx/huanbao.js +++ b/lib/v2/bjx/huanbao.js @@ -33,7 +33,7 @@ module.exports = async (ctx) => { // 服务器禁止单个IP大并发访问，只能少返回几条 3, items, - (items) => fetchPage(ctx, items.link) + (items) => fetchPage(items.link) ); ctx.set('data', { @@ -43,7 +43,7 @@ module.exports = async (ctx) => { }); }; -const fetchPage = (ctx, link) => +const fetchPage = (link) => cache.tryGet(link, async () => { // 可能一篇文章过长会分成多页 const pages = []; diff --git a/lib/v2/dlnews/category.js b/lib/v2/dlnews/category.js index 7b36917550..b8fc63f7af 100644 --- a/lib/v2/dlnews/category.js +++ b/lib/v2/dlnews/category.js @@ -17,7 +17,7 @@ const topics = { snapshot: 'Snapshot', web3: 'Web3', }; -const extractArticle = (ctx, item) => +const extractArticle = (item) => cache.tryGet(item.link, async () => { const { data: response } = await got(item.link); const $ = load(response); @@ -75,7 +75,7 @@ module.exports = async (ctx) => { const data = await getData(`${baseUrl}${apiPath}?query=${encodeURIComponent(JSON.stringify(query))}&_website=${_website}`); const list = getList(data); const items = []; - for await (const data of asyncPool(3, list, (item) => extractArticle(ctx, item))) { + for await (const data of asyncPool(3, list, (item) => extractArticle(item))) { items.push(data); } diff --git a/lib/v2/douban/people/status.js b/lib/v2/douban/people/status.js index ba023fe440..8d753e0300 100644 --- a/lib/v2/douban/people/status.js +++ b/lib/v2/douban/people/status.js @@ -403,15 +403,15 @@ function getContentByActivity(ctx, item, params = {}, picsPrefixes = []) { return { title, description }; } -async function getFullTextItems(ctx, items) { +async function getFullTextItems(items) { const prefix = 'https://m.douban.com/rexxar/api/v2/status/'; await Promise.all( items.map(async (item) => { let url = prefix + item.status.id; - let cache = await cache.get(url); - if (cache) { - item.status.text = cache; + let cacheResult = await cache.get(url); + if (cacheResult) { + item.status.text = cacheResult; } else { const { data: { text }, @@ -424,9 +424,9 @@ async function getFullTextItems(ctx, items) { return; } url = prefix + item.status.reshared_status.id; - cache = await cache.get(url); - if (cache) { - item.status.reshared_status.text = cache; + cacheResult = await cache.get(url); + if (cacheResult) { + item.status.reshared_status.text = cacheResult; } else if (tryFixStatus(item.status.reshared_status).isFixSuccess) { try { // 存在reshared_status字段正常，但尝试获取时返回403的情况。比如原po被炸号就可能这样。 @@ -457,7 +457,7 @@ module.exports = async (ctx) => { ); if (items) { - await getFullTextItems(ctx, items); + await getFullTextItems(items); } ctx.set('data', { diff --git a/lib/v2/eleduck/posts.js b/lib/v2/eleduck/posts.js index e4212c43b1..9985ef2956 100644 --- a/lib/v2/eleduck/posts.js +++ b/lib/v2/eleduck/posts.js @@ -1,7 +1,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; -const getCateName = async (ctx, cid = 0) => { +const getCateName = async (cid = 0) => { const key = 'eleduck-categories'; const cates = await cache.tryGet(key, async () => { const res = await got(`https://svc.eleduck.com/api/v1/categories`); @@ -25,7 +25,7 @@ module.exports = async (ctx) => { throw new Error('没有获取到数据'); } - const cname = await getCateName(ctx, cid); + const cname = await getCateName(cid); ctx.set('data', { title: `电鸭社区的文章--${cname}`, diff --git a/lib/v2/gov/moa/moa.js b/lib/v2/gov/moa/moa.js index f80fe742d0..f0a32dc859 100644 --- a/lib/v2/gov/moa/moa.js +++ b/lib/v2/gov/moa/moa.js @@ -15,7 +15,7 @@ module.exports = async (ctx) => { // 图片新闻 ctx.set( 'data', - await dealChannel(ctx, suburl, { + await dealChannel(suburl, { channelTitleSelector: '.pub-media2-head', listSelector: '.tupian_list li', titleSelector: 'a[class="block w_fill ellipsis adc ahc"]', @@ -26,7 +26,7 @@ module.exports = async (ctx) => { // 公开公告 ctx.set( 'data', - await dealChannel(ctx, 'govpublic/1/index.htm', { + await dealChannel('govpublic/1/index.htm', { channelTitleSelector: 'title', listSelector: '.gongkai_centerRList li', titleSelector: 'a', @@ -36,7 +36,7 @@ module.exports = async (ctx) => { } else { ctx.set( 'data', - await dealChannel(ctx, suburl, { + await dealChannel(suburl, { channelTitleSelector: '.pub-media1-head-title', listSelector: '.ztlb', titleSelector: 'a', @@ -47,7 +47,7 @@ module.exports = async (ctx) => { }; // 处理文章列表, 从那里获得一堆要爬取的页面, 然后爬取 -async function dealChannel(ctx, suburl, selectors) { +async function dealChannel(suburl, selectors) { const { channelTitleSelector, listSelector, titleSelector, dateSelector } = selectors; // 为了与下面解析相对链接的dealLink配合, 这里末尾必须保证有一条斜杠 diff --git a/lib/v2/gov/pbc/zcyj.js b/lib/v2/gov/pbc/zcyj.js index 0e88fb73c3..7622fe528a 100644 --- a/lib/v2/gov/pbc/zcyj.js +++ b/lib/v2/gov/pbc/zcyj.js @@ -19,7 +19,7 @@ module.exports = async (ctx) => { })) .get(); - const items = await processItems(list, ctx); + const items = await processItems(list); ctx.set('data', { title: '中国人民银行 政策研究', diff --git a/lib/v2/gov/sichuan/deyang/govpublicinfo.js b/lib/v2/gov/sichuan/deyang/govpublicinfo.js index 975dfb6f6b..65b58b7165 100644 --- a/lib/v2/gov/sichuan/deyang/govpublicinfo.js +++ b/lib/v2/gov/sichuan/deyang/govpublicinfo.js @@ -37,7 +37,7 @@ const getInfoUrlList = async (rootUrl, infoBasicUrl) => { }; // 获取信息正文内容 -const getInfoContent = (ctx, rootUrl, item) => +const getInfoContent = (rootUrl, item) => cache.tryGet(item.url, async () => { const response = await got(item.url); // 部分网页会跳转其他类型网站,则不解析，直接附超链接 @@ -78,7 +78,7 @@ module.exports = async (ctx) => { const infoBasicUrl = basicInfoDict[countyName].infoType[infoType].basicUrl; const rootUrl = basicInfoDict[countyName].rootUrl; const infoUrlList = await getInfoUrlList(rootUrl, infoBasicUrl); - const items = await Promise.all(infoUrlList.map((item) => getInfoContent(ctx, rootUrl, item))); + const items = await Promise.all(infoUrlList.map((item) => getInfoContent(rootUrl, item))); ctx.set('data', { title: `政府公开信息-${countyName}-${basicInfoDict[countyName].infoType[infoType].name}`, diff --git a/lib/v2/gov/sichuan/deyang/mztoday.js b/lib/v2/gov/sichuan/deyang/mztoday.js index e178930797..43b0219bc9 100644 --- a/lib/v2/gov/sichuan/deyang/mztoday.js +++ b/lib/v2/gov/sichuan/deyang/mztoday.js @@ -104,7 +104,7 @@ const getInfoUrlList = async (url) => { }; // 获取信息正文内容 -const getInfoContent = (ctx, item) => +const getInfoContent = (item) => cache.tryGet(item.url, async () => { const response = await got(item.url); const $ = load(response.data); @@ -120,7 +120,7 @@ module.exports = async (ctx) => { const infoType = ctx.req.param('infoType') || 'zx'; const infoBasicUrl = `${rootUrl}${basicInfoDict[infoType].url}`; const infoUrlList = await getInfoUrlList(infoBasicUrl); - const items = await Promise.all(infoUrlList.map((item) => getInfoContent(ctx, item))); + const items = await Promise.all(infoUrlList.map((item) => getInfoContent(item))); ctx.set('data', { title: `今日绵竹-${basicInfoDict[infoType].name}`, diff --git a/lib/v2/guokr/channel.js b/lib/v2/guokr/channel.js index 4a338415d3..715ec4f85a 100644 --- a/lib/v2/guokr/channel.js +++ b/lib/v2/guokr/channel.js @@ -1,7 +1,7 @@ import cache from '@/utils/cache'; import got from '@/utils/got'; -async function loadFullPage(ctx, id) { +async function loadFullPage(id) { const link = `https://apis.guokr.com/minisite/article/${id}.json`; const content = await cache.tryGet(link, async () => { const res = await got(link); @@ -33,7 +33,7 @@ module.exports = async (ctx) => { const result = await Promise.all( items.map(async (item) => ({ title: item.title, - description: await loadFullPage(ctx, item.id), // Mercury 无法正确解析全文，故这里手动加载 + description: await loadFullPage(item.id), // Mercury 无法正确解析全文，故这里手动加载 pubDate: item.date_published, link: item.url, author: item.author.nickname, diff --git a/lib/v2/hoyolab/news.js b/lib/v2/hoyolab/news.js index 7094cf3451..dcea080f9a 100644 --- a/lib/v2/hoyolab/news.js +++ b/lib/v2/hoyolab/news.js @@ -27,7 +27,7 @@ const getEventList = async ({ type, gids, size, language }) => { const replaceImgDomain = (content) => content.replaceAll(PRIVATE_IMG, PUBLIC_IMG); -const getPostContent = (ctx, list, { language }) => +const getPostContent = (list, { language }) => Promise.all( list.map(async (row) => { const post = row.post; @@ -82,7 +82,7 @@ module.exports = async (ctx) => { const gameInfo = await getI18nGameInfo(gids, language, cache.tryGet); const typeInfo = await getI18nType(language, cache.tryGet); const list = await getEventList(params); - const items = await getPostContent(ctx, list, params); + const items = await getPostContent(list, params); ctx.set('data', { title: `HoYoLAB-${gameInfo.name}-${typeInfo[type].title}`, link: `${LINK}/circles/${gids}/${OFFICIAL_PAGE_TYPE[gids]}/official?page_type=${OFFICIAL_PAGE_TYPE[gids]}&page_sort=${typeInfo[type].sort}`, diff --git a/lib/v2/kuaidi100/index.js b/lib/v2/kuaidi100/index.js index 468b424625..3c06812cf7 100644 --- a/lib/v2/kuaidi100/index.js +++ b/lib/v2/kuaidi100/index.js @@ -8,14 +8,14 @@ module.exports = async (ctx) => { // I am doing these to avoid invaild request. // First, check if code is vaild - const { status, message, company } = await utils.checkCode(ctx, number, id, phone); + const { status, message, company } = await utils.checkCode(number, id, phone); let data; let query; const time = new Date().toString(); if (status) { - query = await utils.getQuery(ctx, number, id, phone); + query = await utils.getQuery(number, id, phone); data = query.status === '200' ? query.data diff --git a/lib/v2/kuaidi100/supported-company.js b/lib/v2/kuaidi100/supported-company.js index 247ebe1508..c739377c9b 100644 --- a/lib/v2/kuaidi100/supported-company.js +++ b/lib/v2/kuaidi100/supported-company.js @@ -1,7 +1,7 @@ const utils = require('./utils'); module.exports = async (ctx) => { - const ls = await utils.company(ctx); + const ls = await utils.company(); ctx.set('data', { title: `快递100 快递列表`, link: 'https://www.kuaidi100.com', diff --git a/lib/v2/kuaidi100/utils.js b/lib/v2/kuaidi100/utils.js index a353d4c398..71ec748f9c 100644 --- a/lib/v2/kuaidi100/utils.js +++ b/lib/v2/kuaidi100/utils.js @@ -9,9 +9,9 @@ const dasddocHref_key = 'kuaidi100-dasddocHref'; const query_count = 'kuaidi100-cookie-count'; const max_query_count = 30; -async function getCookie(ctx) { +async function getCookie() { // Check if this key should be replace? every 30 times should be fine. - shouldUpdateCookie(ctx); + shouldUpdateCookie(); let wwwid = await cache.get(wwwid_key); let csrf = await cache.get(csrf_key); let globacsrftoken = await cache.get(globacsrftoken_key); @@ -55,7 +55,7 @@ async function getCookie(ctx) { cache.set(dasddocHref_key, dasddocHref, 600); // We have acquired new cookie. It may due to cache timeout. // Force update counter and don't wait it finished. - shouldUpdateCookie(ctx, true); + shouldUpdateCookie(true); } return { @@ -87,11 +87,11 @@ async function getCookie(ctx) { "version": 776 } */ -function getCompanyList(ctx) { +function getCompanyList() { // Using date as cache key and it will automatically expired by 1d const key = `kuaidi100-company-name-${new Date().toISOString().split('T')[0]}`; return cache.tryGet(key, async () => { - const cookie = await getCookie(ctx); + const cookie = await getCookie(); const wwwid = cookie.wwwid; const companyResponse = await got({ method: 'post', @@ -116,7 +116,7 @@ function getCompanyList(ctx) { }); } -function shouldUpdateCookie(ctx, forcedUpdate = false) { +function shouldUpdateCookie(forcedUpdate = false) { if (forcedUpdate) { cache.set(query_count, 0); } else { @@ -140,9 +140,9 @@ function clearCookie() { } module.exports = { - company: (ctx) => getCompanyList(ctx), - checkCode: async (ctx, number, id, phone) => { - const list = await getCompanyList(ctx); + company: () => getCompanyList(), + checkCode: async (number, id, phone) => { + const list = await getCompanyList(); const company = list.find((c) => c.number === number); if (company) { if (number.includes('shunfeng') && !isNaN(phone) && String(phone).length !== 4) { @@ -205,7 +205,7 @@ module.exports = { } */ - getQuery: async (ctx, number, id, phone) => { + getQuery: async (number, id, phone) => { const query_key = `kuaidi100-query-${number}-${id}`; let query = await cache.get(query_key); // 组装日期与单号 日期为提前两个月 @@ -222,7 +222,7 @@ module.exports = { if (query) { query = JSON.parse(query); } else { - const cookie = await getCookie(ctx); + const cookie = await getCookie(); const queryResponse = await got({ method: 'get', url: `https://www.kuaidi100.com/query?type=${number}&postid=${id}&temp=${Math.random()}&phone=${phone ?? ''}`, diff --git a/lib/v2/liveuamap/index.js b/lib/v2/liveuamap/index.js index 7ddd96eaec..9e6b8210c8 100644 --- a/lib/v2/liveuamap/index.js +++ b/lib/v2/liveuamap/index.js @@ -3,17 +3,13 @@ import { load } from 'cheerio'; import { isValidHost } from '@/utils/valid-host'; module.exports = async (ctx) => { - let region = ctx.req.param('region') ?? 'ukraine'; + const region = ctx.req.param('region') ?? 'ukraine'; const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 50; if (!isValidHost(region)) { throw new Error('Invalid region'); } - let url = `https://${region}.liveuamap.com/`; - if (region === undefined) { - region = 'Default'; - url = 'https://liveuamap.com/'; - } + const url = `https://${region}.liveuamap.com/`; const response = await got({ method: 'get', diff --git a/lib/v2/newrank/utils.js b/lib/v2/newrank/utils.js index 26364abf10..0e947772e9 100644 --- a/lib/v2/newrank/utils.js +++ b/lib/v2/newrank/utils.js @@ -44,7 +44,7 @@ const decrypt_douyin_detail_xyz = (nonce) => { const flatten = (arr) => arr.reduce((acc, val) => [...acc, ...(Array.isArray(val) ? flatten(val) : val)], []); -function shouldUpdateCookie(ctx, forcedUpdate = false) { +function shouldUpdateCookie(forcedUpdate = false) { if (forcedUpdate) { cache.set(query_count, 0); } else { @@ -52,7 +52,7 @@ function shouldUpdateCookie(ctx, forcedUpdate = false) { if (count) { if (count > max_query_count) { cache.set(query_count, 0); - clearCookie(ctx); + clearCookie(); } else { cache.set(query_count, count + 1); } @@ -67,9 +67,9 @@ function clearCookie() { } // 加了验证码失效了 -async function getCookie(ctx) { +async function getCookie() { // Check if this key should be replace? every 30 times should be fine. - shouldUpdateCookie(ctx); + shouldUpdateCookie(); let token = await cache.get(newrank_cookie_token); const username = String(config.newrank.username); const password = md5(md5(String(config.newrank.password)) + 'daddy'); @@ -98,7 +98,7 @@ async function getCookie(ctx) { cache.set(newrank_cookie_token, token, 600); // We have acquired new cookie. It may due to cache timeout. // Force update counter and don't wait it finished. - shouldUpdateCookie(ctx, true); + shouldUpdateCookie(true); } return token; } diff --git a/lib/v2/phoronix/index.js b/lib/v2/phoronix/index.js index 60bac5ce6e..9fd4c80f36 100644 --- a/lib/v2/phoronix/index.js +++ b/lib/v2/phoronix/index.js @@ -64,7 +64,7 @@ const webFetchCb = (response) => { }; }; -const webFetch = (ctx, url) => +const webFetch = (url) => cache.tryGet(`${webArticlesCacheKey}:${url}`, async () => { try { return webFetchCb(await got(url)); @@ -76,7 +76,7 @@ const webFetch = (ctx, url) => } }); -const legacyFetch = async (ctx, page, queryOrItem) => { +const legacyFetch = async (page, queryOrItem) => { const legacyUrl = new URL('/scan.php', baseUrl); legacyUrl.searchParams.set('page', page); if (queryOrItem) { @@ -97,14 +97,14 @@ const legacyFetch = async (ctx, page, queryOrItem) => { cache.set(`${webArticlesCacheKey}:${webUrl}`, feed); return feed; } - return await webFetch(ctx, webUrl); + return await webFetch(webUrl); }; -const tryFetch = async (ctx, category, topic) => { +const tryFetch = async (category, topic) => { const webUrl = topic ? `${baseUrl}/${category}/${topic}` : `${baseUrl}/${category}`; - let feed = await webFetch(ctx, webUrl); + let feed = await webFetch(webUrl); if (feed === '404') { - feed = await legacyFetch(ctx, category, topic); + feed = await legacyFetch(category, topic); } return feed; }; @@ -115,13 +115,13 @@ module.exports = async (ctx) => { switch (category) { case 'category': case 'news_topic': - feed = await legacyFetch(ctx, category, topic); + feed = await legacyFetch(category, topic); break; case 'rss': feed = await feedFetch(); break; default: - feed = category ? await tryFetch(ctx, category, topic) : await feedFetch(); + feed = category ? await tryFetch(category, topic) : await feedFetch(); break; } diff --git a/lib/v2/sensortower/blog.js b/lib/v2/sensortower/blog.js index c83e35701f..19122b5f32 100644 --- a/lib/v2/sensortower/blog.js +++ b/lib/v2/sensortower/blog.js @@ -65,6 +65,6 @@ module.exports = async (ctx) => { title: 'Sensor Tower - Blog', link: currentUrl, item: items, - language: language ?? 'en-US', + language: language || 'en-US', }); }; diff --git a/lib/v2/udn/breaking-news.js b/lib/v2/udn/breaking-news.js index f125c4a177..d39bf52e54 100644 --- a/lib/v2/udn/breaking-news.js +++ b/lib/v2/udn/breaking-news.js @@ -9,7 +9,7 @@ import * as path from 'node:path'; module.exports = async (ctx) => { const id = ctx.req.param('id'); const link = `/news/breaknews/1/${id}#breaknews`; - const name = await getLinkName(ctx, link); + const name = await getLinkName(link); const rootUrl = 'https://udn.com'; const response = await got(`${rootUrl}/api/more?page=1&channelId=1&cate_id=${id}&type=breaknews`); @@ -75,7 +75,7 @@ module.exports = async (ctx) => { }); }; -const getLinkName = async (ctx, link) => { +const getLinkName = async (link) => { const url = 'https://udn.com/news/breaknews'; const links = await cache.tryGet(url, async () => { const result = await got(url); diff --git a/lib/v2/wechat/tgchannel.js b/lib/v2/wechat/tgchannel.js index 2a5fc0ecd8..931e52a9b2 100644 --- a/lib/v2/wechat/tgchannel.js +++ b/lib/v2/wechat/tgchannel.js @@ -138,7 +138,7 @@ module.exports = async (ctx) => { out.reverse(); ctx.set('data', { - title: mpName ?? $('.tgme_channel_info_header_title').text(), + title: $('.tgme_channel_info_header_title').text(), link: `https://t.me/s/${id}`, item: out.filter(Boolean), allowEmpty: !!mpName, diff --git a/lib/v2/xiaoyuzhou/pickup.js b/lib/v2/xiaoyuzhou/pickup.js index e92be7c215..4cfacb64a1 100644 --- a/lib/v2/xiaoyuzhou/pickup.js +++ b/lib/v2/xiaoyuzhou/pickup.js @@ -79,10 +79,10 @@ const ProcessFeed = async () => { }; module.exports = async (ctx) => { - let resultItems = await cache.tryGet(XIAOYUZHOU_ITEMS, () => ProcessFeed(ctx)); + let resultItems = await cache.tryGet(XIAOYUZHOU_ITEMS, () => ProcessFeed()); if (!isToday(resultItems[0].pubDate)) { // force refresh cache - resultItems = await ProcessFeed(ctx); + resultItems = await ProcessFeed(); cache.set(XIAOYUZHOU_ITEMS, resultItems); } ctx.set('data', {