opensource/RSSHub
3
0
Fork 0
mirror of https://github.com/DIYgod/RSSHub.git synced 2026-08-29 01:53:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: replace tiny-async-pool with p-map (#18928)

Browse Source 
* refactor: replace tiny-async-pool with p-map

* fix: use original concurrency
This commit is contained in:
Tony
2025-04-23 21:02:10 +08:00
committed by GitHub
parent 78757950aa
commit df320988d0
38 changed files with 483 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/routes/agefans/update.ts
+20 -20
View File
@@ -1,9 +1,9 @@
import { Route } from '@/types';
import { DataItem, Route } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { rootUrl } from './utils';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/update',
@@ -47,27 +47,27 @@ async function handler() {
};
});
const items: any[] = [];
for await (const item of asyncPool(3, list, (item) =>
cache.tryGet(item.link, async () => {
const detailResponse = await got(item.link);
const content = load(detailResponse.data);
const items: DataItem[] = await pMap(
list,
(item) =>
cache.tryGet(item.link, async () => {
const detailResponse = await got(item.link);
const content = load(detailResponse.data);
content('img').each((_, ele) => {
if (ele.attribs['data-original']) {
ele.attribs.src = ele.attribs['data-original'];
delete ele.attribs['data-original'];
}
});
content('.video_detail_collect').remove();
content('img').each((_, ele) => {
if (ele.attribs['data-original']) {
ele.attribs.src = ele.attribs['data-original'];
delete ele.attribs['data-original'];
}
});
content('.video_detail_collect').remove();
item.description = content('.video_detail_left').html();
item.description = content('.video_detail_left').html();
return item;
})
)) {
items.push(item);
}
return item;
}),
{ concurrency: 3 }
);
return {
title: $('title').text(),
lib/routes/apnews/mobile-api.ts
+3 -2
View File
@@ -1,5 +1,6 @@
import { Route, ViewType } from '@/types';
import { asyncPoolAll, fetchArticle } from './utils';
import { fetchArticle } from './utils';
import pMap from 'p-map';
import ofetch from '@/utils/ofetch';
import { parseDate } from '@/utils/parse-date';
@@ -83,7 +84,7 @@ async function handler(ctx) {
.sort((a, b) => b.pubDate - a.pubDate)
.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
const items = ctx.req.query('fulltext') === 'true' ? await asyncPoolAll(10, list, (item) => fetchArticle(item)) : list;
const items = ctx.req.query('fulltext') === 'true' ? await pMap(list, (item) => fetchArticle(item), { concurrency: 10 }) : list;
return {
title: screen.category ?? screen.title,
lib/routes/apnews/sitemap.ts
+3 -2
View File
@@ -1,5 +1,6 @@
import { Route, ViewType } from '@/types';
import { asyncPoolAll, fetchArticle } from './utils';
import { fetchArticle } from './utils';
import pMap from 'p-map';
import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
@@ -81,7 +82,7 @@ async function handler(ctx) {
.sort((a, b) => (a.pubDate && b.pubDate ? b.pubDate - a.pubDate : b.lastmod - a.lastmod))
.slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20);
const items = ctx.req.query('fulltext') === 'true' ? await asyncPoolAll(20, list, (item) => fetchArticle(item)) : list;
const items = ctx.req.query('fulltext') === 'true' ? await pMap(list, (item) => fetchArticle(item), { concurrency: 20 }) : list;
return {
title: `AP News sitemap:${route}`,
lib/routes/apnews/topics.ts
+3 -2
View File
@@ -1,7 +1,8 @@
import { Route, ViewType } from '@/types';
import got from '@/utils/got';
import { load } from 'cheerio';
import { asyncPoolAll, fetchArticle, removeDuplicateByKey } from './utils';
import { fetchArticle, removeDuplicateByKey } from './utils';
import pMap from 'p-map';
const HOME_PAGE = 'https://apnews.com';
export const route: Route = {
@@ -50,7 +51,7 @@ async function handler(ctx) {
}))
.filter((e) => typeof e.link === 'string');
const items = ctx.req.query('fulltext') === 'true' ? await asyncPoolAll(10, list, (item) => fetchArticle(item)) : list;
const items = ctx.req.query('fulltext') === 'true' ? await pMap(list, (item) => fetchArticle(item), { concurrency: 10 }) : list;
return {
title: $('title').text(),
lib/routes/apnews/utils.ts
-8
View File
@@ -2,7 +2,6 @@ import cache from '@/utils/cache';
import ofetch from '@/utils/ofetch';
import { parseDate } from '@/utils/parse-date';
import { load } from 'cheerio';
import asyncPool from 'tiny-async-pool';
export function removeDuplicateByKey(items, key: string) {
return [...new Map(items.map((x) => [x[key], x])).values()];
@@ -65,10 +64,3 @@ export function fetchArticle(item) {
}
});
}
export async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
const results: Awaited<OUT[]> = [];
for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
results.push(result);
}
return results;
}
lib/routes/bdys/index.ts
+32 -33
View File
@@ -7,7 +7,7 @@ import { parseDate } from '@/utils/parse-date';
import timezone from '@/utils/timezone';
import { art } from '@/utils/render';
import path from 'node:path';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
import { config } from '@/config';
import ConfigNotFoundError from '@/errors/types/config-not-found';
@@ -139,43 +139,42 @@ async function handler(ctx) {
cookie: `JSESSIONID=${jsessionid}`,
};
const items = [];
const items = await pMap(
list,
(item) =>
cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
headers,
});
const downloadResponse = await got({
method: 'get',
url: `${rootUrl}/downloadInfo/list?mid=${item.link.split('/')[4].split('.')[0]}`,
headers,
});
const content = load(detailResponse.data);
for await (const data of asyncPool(1, list, (item) =>
cache.tryGet(item.link, async () => {
const detailResponse = await got({
method: 'get',
url: item.link,
headers,
});
const downloadResponse = await got({
method: 'get',
url: `${rootUrl}/downloadInfo/list?mid=${item.link.split('/')[4].split('.')[0]}`,
headers,
});
const content = load(detailResponse.data);
content('svg').remove();
const torrents = content('.download-list .list-group');
content('svg').remove();
const torrents = content('.download-list .list-group');
item.description = art(path.join(__dirname, 'templates/desc.art'), {
info: content('.row.mt-3').html(),
synopsis: content('#synopsis').html(),
links: downloadResponse.data,
torrents: torrents.html(),
});
item.description = art(path.join(__dirname, 'templates/desc.art'), {
info: content('.row.mt-3').html(),
synopsis: content('#synopsis').html(),
links: downloadResponse.data,
torrents: torrents.html(),
});
item.pubDate = timezone(parseDate(content('.bg-purple-lt').text().replace('更新时间：', '')), +8);
item.guid = `${item.link}#${content('.card h1').text()}`;
item.pubDate = timezone(parseDate(content('.bg-purple-lt').text().replace('更新时间：', '')), +8);
item.guid = `${item.link}#${content('.card h1').text()}`;
item.enclosure_url = torrents.html() ? `${rootUrl}${torrents.find('a').first().attr('href')}` : downloadResponse.data.pop().url;
item.enclosure_type = 'application/x-bittorrent';
item.enclosure_url = torrents.html() ? `${rootUrl}${torrents.find('a').first().attr('href')}` : downloadResponse.data.pop().url;
item.enclosure_type = 'application/x-bittorrent';
return item;
})
)) {
items.push(data);
}
return item;
}),
{ concurrency: 1 }
);
return {
title: '哔嘀影视',
lib/routes/bjnews/cat.ts
+2 -9
View File
@@ -3,7 +3,7 @@ import { load } from 'cheerio';
import ofetch from '@/utils/ofetch';
import { fetchArticle } from './utils';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/cat/:cat',
@@ -35,17 +35,10 @@ async function handler(ctx) {
category: $(a).parent().find('.source').text().trim(),
}));
const out = await asyncPoolAll(2, list, (item) => fetchArticle(item));
const out = await pMap(list, (item) => fetchArticle(item), { concurrency: 2 });
return {
title: `新京报 - 分类 - ${$('.cur').text().trim()}`,
link: url,
item: out,
};
}
async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
const results: Awaited<OUT[]> = [];
for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
results.push(result);
}
return results;
}
lib/routes/bjx/huanbao.ts
+4 -12
View File
@@ -4,15 +4,7 @@ import got from '@/utils/got';
import { parseDate } from '@/utils/parse-date';
import { load } from 'cheerio';
import timezone from '@/utils/timezone';
import asyncPool from 'tiny-async-pool';
const asyncPoolAll = async (...args) => {
const results = [];
for await (const result of asyncPool(...args)) {
results.push(result);
}
return results;
};
import pMap from 'p-map';
export const route: Route = {
path: '/huanbao',
@@ -54,11 +46,11 @@ async function handler() {
};
});
items = await asyncPoolAll(
items = await pMap(
// 服务器禁止单个IP大并发访问，只能少返回几条
3,
items,
(items) => fetchPage(items.link)
(item) => fetchPage(item.link),
{ concurrency: 3 }
);
return {
lib/routes/bloomberg/authors.ts
+3 -2
View File
@@ -2,7 +2,8 @@ import { Route, ViewType } from '@/types';
import { load } from 'cheerio';
import ofetch from '@/utils/ofetch';
import rssParser from '@/utils/rss-parser';
import { asyncPoolAll, parseArticle } from './utils';
import { parseArticle } from './utils';
import pMap from 'p-map';
const parseAuthorNewsList = async (slug) => {
const baseURL = `https://www.bloomberg.com/authors/${slug}`;
@@ -66,7 +67,7 @@ async function handler(ctx) {
list = (await rssParser.parseURL(`${link}.rss`)).items;
}
const item = await asyncPoolAll(1, list, (item) => parseArticle(item));
const item = await pMap(list, (item) => parseArticle(item), { concurrency: 1 });
const authorName = item.find((i) => i.author)?.author ?? slug;
return {
lib/routes/bloomberg/index.ts
+6 -5
View File
@@ -1,6 +1,7 @@
import { Route, ViewType } from '@/types';
import { rootUrl, asyncPoolAll, parseNewsList, parseArticle } from './utils';
const site_title_mapping = {
import { rootUrl, parseNewsList, parseArticle } from './utils';
import pMap from 'p-map';
const siteTitleMapping = {
'/': 'News',
bpol: 'Politics',
bbiz: 'Business',
@@ -23,7 +24,7 @@ export const route: Route = {
parameters: {
site: {
description: 'Site ID, can be found below',
options: Object.keys(site_title_mapping).map((key) => ({ value: key, label: site_title_mapping[key] })),
options: Object.keys(siteTitleMapping).map((key) => ({ value: key, label: siteTitleMapping[key] })),
},
},
features: {
@@ -60,9 +61,9 @@ async function handler(ctx) {
const currentUrl = site ? `${rootUrl}/${site}/sitemap_news.xml` : `${rootUrl}/sitemap_news.xml`;
const list = await parseNewsList(currentUrl, ctx);
const items = await asyncPoolAll(1, list, (item) => parseArticle(item));
const items = await pMap(list, (item) => parseArticle(item), { concurrency: 1 });
return {
title: `Bloomberg - ${site_title_mapping[site ?? '/']}`,
title: `Bloomberg - ${siteTitleMapping[site ?? '/']}`,
link: currentUrl,
item: items,
};
lib/routes/bloomberg/utils.ts
+1 -10
View File
@@ -1,7 +1,6 @@
import cache from '@/utils/cache';
import { load } from 'cheerio';
import path from 'node:path';
import asyncPool from 'tiny-async-pool';
import { destr } from 'destr';
import { parseDate } from '@/utils/parse-date';
@@ -604,12 +603,4 @@ const documentToHtmlString = async (document) => {
return str;
};
const asyncPoolAll = async (...args) => {
const results = [];
for await (const result of asyncPool(...args)) {
results.push(result);
}
return results;
};
export { rootUrl, asyncPoolAll, parseNewsList, parseArticle };
export { rootUrl, parseNewsList, parseArticle };
lib/routes/cara/utils.ts
-9
View File
@@ -1,7 +1,6 @@
import { config } from '@/config';
import ofetch from '@/utils/ofetch';
import type { FetchOptions, FetchRequest, ResponseType } from 'ofetch';
import asyncPool from 'tiny-async-pool';
import type { PortfolioDetailResponse, PortfolioResponse, UserNextData } from './types';
import type { DataItem } from '@/types';
import { parseDate } from '@/utils/parse-date';
@@ -35,14 +34,6 @@ export async function parseUserData(user: string) {
})) as Promise<UserNextData['pageProps']['user']>;
}
export async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
const results: Awaited<OUT[]> = [];
for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
results.push(result);
}
return results;
}
export async function fetchPortfolioItem(item: PortfolioResponse['data'][number]) {
const res = await customFetch<PortfolioDetailResponse>(`${API_HOST}/posts/${item.postId}`);
lib/routes/cfr/index.ts
+3 -2
View File
@@ -2,7 +2,8 @@ import type { Data, Route } from '@/types';
import type { Context } from 'hono';
import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import { asyncPoolAll, getDataItem } from './utils';
import { getDataItem } from './utils';
import pMap from 'p-map';
export const route: Route = {
path: '/:category/:subCategory?',
@@ -48,7 +49,7 @@ async function handler(ctx: Context): Promise<Data> {
const listSelector = selectorMap[category] ?? '.card-article-large__link';
const items = await asyncPoolAll(5, $(listSelector).toArray(), async (item) => await getDataItem($(item).attr('href')!));
const items = await pMap($(listSelector).toArray(), (item) => getDataItem($(item).attr('href')!), { concurrency: 5 });
return {
title: $('head title').text().replace(' | Council on Foreign Relations', ''),
lib/routes/cfr/utils.ts
-9
View File
@@ -4,7 +4,6 @@ import type { DataItem } from '@/types';
import { parseDate } from '@/utils/parse-date';
import cache from '@/utils/cache';
import type { LinkData, VideoSetup } from './types';
import asyncPool from 'tiny-async-pool';
export function getDataItem(href: string) {
const origin = 'https://www.cfr.org';
@@ -274,11 +273,3 @@ function parseDescription($description: Cheerio<Element>, $: CheerioAPI) {
return description;
}
export async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
const results: Awaited<OUT[]> = [];
for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
results.push(result);
}
return results;
}
lib/routes/copymanga/comic.ts
+2 -10
View File
@@ -7,7 +7,7 @@ import { parseDate } from '@/utils/parse-date';
import { art } from '@/utils/render';
import path from 'node:path';
import { config } from '@/config';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/comic/:id/:chapterCnt?',
@@ -126,15 +126,7 @@ async function handler(ctx) {
};
};
const asyncPoolAll = async (...args) => {
const results = [];
for await (const result of asyncPool(...args)) {
results.push(result);
}
return results;
};
const result = await asyncPoolAll(3, chapterArray.slice(0, chapterCnt), (chapter) => cache.tryGet(chapter.link, () => genResult(chapter)));
const result = await pMap(chapterArray.slice(0, chapterCnt), (chapter) => cache.tryGet(chapter.link, () => genResult(chapter)), { concurrency: 3 });
const items = [...result, ...chapterArray.slice(chapterCnt)];
return {
lib/routes/cpta/handler.ts
+2 -6
View File
@@ -2,7 +2,7 @@ import { DataItem, Route } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
type NewsCategory = {
title: string;
@@ -77,11 +77,7 @@ const handler: Route['handler'] = async (ctx) => {
} as DataItem;
});
const dataItems: DataItem[] = [];
for await (const item of await asyncPool(1, contentLinkList, fetchDataItem)) {
dataItems.push(item as DataItem);
}
const dataItems: DataItem[] = await pMap(contentLinkList, fetchDataItem, { concurrency: 1 });
return {
title: `中国人事考试网-${NEWS_TYPES[category].title}`,
lib/routes/cts/news.ts
+23 -22
View File
@@ -3,7 +3,7 @@ import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/:category',
@@ -36,28 +36,29 @@ async function handler(ctx) {
const currentUrl = `https://news.cts.com.tw/${category}/index.html`;
const response = await got(currentUrl);
const $ = load(response.data);
const items = [];
for await (const data of asyncPool(5, $('#newslist-top a[title]').slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 20), (item) => {
item = $(item);
const link = item.attr('href');
return cache.tryGet(link, async () => {
const response = await got(link);
const $ = load(response.data);
const author = $('.artical-content p:eq(0)').text().trim();
$('.artical-content p:eq(0), .artical-content .flexbox').remove();
const items = await pMap(
$('#newslist-top a[title]').slice(0, ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 20),
(item) => {
item = $(item);
const link = item.attr('href');
return cache.tryGet(link, async () => {
const response = await got(link);
const $ = load(response.data);
const author = $('.artical-content p:eq(0)').text().trim();
$('.artical-content p:eq(0), .artical-content .flexbox').remove();
return {
title: item.attr('title'),
author,
description: $('.artical-content').html(),
category: $('meta[property="article:section"]').attr('content'),
pubDate: parseDate($('meta[property="article:published_time"]').attr('content')),
link,
};
});
})) {
items.push(data);
}
return {
title: item.attr('title'),
author,
description: $('.artical-content').html(),
category: $('meta[property="article:section"]').attr('content'),
pubDate: parseDate($('meta[property="article:published_time"]').attr('content')),
link,
};
});
},
{ concurrency: 5 }
);
return {
title: $('title').text(),
lib/routes/dcard/utils.ts
+38 -37
View File
@@ -1,46 +1,47 @@
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
const ProcessFeed = async (items, cookies, browser, limit, cache) => {
let newCookies = [];
const result = [];
for await (const item of asyncPool(3, items.slice(0, limit), async (i) => {
const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`;
const content = await cache.tryGet(`dcard:${i.id}`, async () => {
let response;
// try catch 处理被删除的帖子
try {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
});
await page.setExtraHTTPHeaders({
referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`,
});
await page.setCookie(...cookies);
await page.goto(url);
await page.waitForSelector('body > pre');
response = await page.evaluate(() => document.querySelector('body > pre').textContent);
newCookies = await page.cookies();
await page.close();
const result = await pMap(
items.slice(0, limit),
async (i) => {
const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`;
const content = await cache.tryGet(`dcard:${i.id}`, async () => {
let response;
// try catch 处理被删除的帖子
try {
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
});
await page.setExtraHTTPHeaders({
referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`,
});
await page.setCookie(...cookies);
await page.goto(url);
await page.waitForSelector('body > pre');
response = await page.evaluate(() => document.querySelector('body > pre').textContent);
newCookies = await page.cookies();
await page.close();
const data = JSON.parse(response);
let body = data.content;
body = body.replaceAll(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => `<img src="${m}">`);
body = body.replaceAll(/(?=https?:\/\/).*(?<!jpe?g"?>?)$/gim, (m) => `<a href="${m}">${m}</a>`);
body = body.replaceAll('\n', '<br>');
const data = JSON.parse(response);
let body = data.content;
body = body.replaceAll(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => `<img src="${m}">`);
body = body.replaceAll(/(?=https?:\/\/).*(?<!jpe?g"?>?)$/gim, (m) => `<a href="${m}">${m}</a>`);
body = body.replaceAll('\n', '<br>');
return body;
} catch {
return '';
}
});
return body;
} catch {
return '';
}
});
i.description = content;
return i;
})) {
result.push(item);
}
i.description = content;
return i;
},
{ concurrency: 3 }
);
await cache.set('dcard:cookies', newCookies, 3600);
return [...result, ...items.slice(limit)];
};
lib/routes/dlnews/category.ts
+2 -5
View File
@@ -6,7 +6,7 @@ import got from '@/utils/got';
import { getData, getList } from './utils';
import { art } from '@/utils/render';
import path from 'node:path';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
const _website = 'dlnews';
const topics = {
@@ -91,10 +91,7 @@ async function handler(ctx) {
};
const data = await getData(`${baseUrl}${apiPath}?query=${encodeURIComponent(JSON.stringify(query))}&_website=${_website}`);
const list = getList(data);
const items = [];
for await (const data of asyncPool(3, list, (item) => extractArticle(item))) {
items.push(data);
}
const items = await pMap(list, (item) => extractArticle(item), { concurrency: 3 });
return {
title: Object.hasOwn(topics, category) ? `${topics[category]} : DL News` : 'DL News',
lib/routes/gdut/oa-news.ts
+85 -84
View File
@@ -5,7 +5,7 @@ import { load } from 'cheerio';
import { CookieJar } from 'tough-cookie';
import { parseDate } from '@/utils/parse-date';
import timezone from '@/utils/timezone';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
const site = 'https://oas.gdut.edu.cn/seeyon';
const typeMap = {
@@ -123,94 +123,95 @@ async function handler(ctx) {
category: item.typeName,
}));
const results = [];
// 获取实际的文章内容
for await (const data of asyncPool(2, articles, async (data) => {
const link = data.link;
data.description = await cache.tryGet(link, async () => {
// 获取数据
const response = await got(link, {
cookieJar,
});
const results = await pMap(
articles,
async (data) => {
const link = data.link;
data.description = await cache.tryGet(link, async () => {
// 获取数据
const response = await got(link, {
cookieJar,
});
const $ = load(response.data);
const node = $('#content');
// 清理样式
node.find('*')
.filter(function () {
return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style';
})
.remove();
node.find('*')
.contents()
.filter(function () {
return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style';
})
.remove();
node.find('*').each(function () {
if (this.attribs.style !== undefined) {
const newSty = this.attribs.style
.split(';')
.filter((s) => {
const styBlocklist = ['color:rgb(0,0,0)', 'color:black', 'background:rgb(255,255,255)', 'background:white', 'text-align:left', 'text-align:justify', 'font-style:normal', 'font-weight:normal'];
const styPrefixBlocklist = [
'font-family',
'font-size',
'background',
'text-autospace',
'text-transform',
'letter-spacing',
'line-height',
'padding',
'margin',
'text-justify',
'word-break',
'vertical-align',
'mso-',
'-ms-',
];
const sty = s.trim();
if (styBlocklist.includes(sty.replaceAll(/\s+/g, ''))) {
return false;
}
for (const prefix of styPrefixBlocklist) {
if (sty.startsWith(prefix)) {
const $ = load(response.data);
const node = $('#content');
// 清理样式
node.find('*')
.filter(function () {
return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style';
})
.remove();
node.find('*')
.contents()
.filter(function () {
return this.type === 'comment' || this.tagName === 'meta' || this.tagName === 'style';
})
.remove();
node.find('*').each(function () {
if (this.attribs.style !== undefined) {
const newSty = this.attribs.style
.split(';')
.filter((s) => {
const styBlocklist = ['color:rgb(0,0,0)', 'color:black', 'background:rgb(255,255,255)', 'background:white', 'text-align:left', 'text-align:justify', 'font-style:normal', 'font-weight:normal'];
const styPrefixBlocklist = [
'font-family',
'font-size',
'background',
'text-autospace',
'text-transform',
'letter-spacing',
'line-height',
'padding',
'margin',
'text-justify',
'word-break',
'vertical-align',
'mso-',
'-ms-',
];
const sty = s.trim();
if (styBlocklist.includes(sty.replaceAll(/\s+/g, ''))) {
return false;
}
}
return true;
})
.join(';');
if (newSty) {
this.attribs.style = newSty;
} else {
delete this.attribs.style;
for (const prefix of styPrefixBlocklist) {
if (sty.startsWith(prefix)) {
return false;
}
}
return true;
})
.join(';');
if (newSty) {
this.attribs.style = newSty;
} else {
delete this.attribs.style;
}
}
}
if (this.attribs.class && this.attribs.class.trim().startsWith('Mso')) {
delete this.attribs.class;
}
if (this.attribs.lang) {
delete this.attribs.lang;
}
if (this.tagName === 'font' || this.tagName === 'o:p') {
$(this).replaceWith(this.childNodes);
}
if (this.tagName === 'span' && !this.attribs.style) {
$(this).replaceWith(this.childNodes);
}
});
node.find('span').each(function () {
if (this.childNodes.length === 0) {
$(this).remove();
}
});
if (this.attribs.class && this.attribs.class.trim().startsWith('Mso')) {
delete this.attribs.class;
}
if (this.attribs.lang) {
delete this.attribs.lang;
}
if (this.tagName === 'font' || this.tagName === 'o:p') {
$(this).replaceWith(this.childNodes);
}
if (this.tagName === 'span' && !this.attribs.style) {
$(this).replaceWith(this.childNodes);
}
});
node.find('span').each(function () {
if (this.childNodes.length === 0) {
$(this).remove();
}
});
return node.html();
});
})) {
results.push(data);
}
return node.html();
});
return data;
},
{ concurrency: 2 }
);
return {
title: `广东工业大学新闻通知网 - ` + type.name,
lib/routes/gov/nrta/dsj.ts
+13 -14
View File
@@ -3,7 +3,7 @@ import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/nrta/dsj/:category?',
@@ -52,23 +52,22 @@ async function handler(ctx) {
};
});
const results = [];
const results = await pMap(
items,
(item) =>
cache.tryGet(item.link, async () => {
const { data: detailResponse } = await got(item.link);
for await (const item of asyncPool(5, items, (item) =>
cache.tryGet(item.link, async () => {
const { data: detailResponse } = await got(item.link);
const content = load(detailResponse);
const content = load(detailResponse);
content('table').last().remove();
content('table').last().remove();
item.description = content('td.newstext').html() || content('table').last().parent().parent().html();
item.description = content('td.newstext').html() || content('table').last().parent().parent().html();
return item;
})
)) {
results.push(item);
}
return item;
}),
{ concurrency: 5 }
);
return {
item: results,
lib/routes/guozaoke/index.ts
+34 -35
View File
@@ -4,7 +4,7 @@ import { load } from 'cheerio';
import { parseRelativeDate } from '@/utils/parse-date';
import { config } from '@/config';
import cache from '@/utils/cache';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/default',
@@ -54,42 +54,41 @@ async function handler() {
})
.filter((item) => item !== undefined);
const out = [];
for await (const result of asyncPool(2, items, (item) =>
cache.tryGet(item.link, async () => {
const url = `https://www.guozaoke.com${item.link}`;
const res = await got({
method: 'get',
url,
headers: {
Cookie: config.guozaoke.cookies,
'User-Agent': config.ua,
},
});
const out = await pMap(
items,
(item) =>
cache.tryGet(item.link, async () => {
const url = `https://www.guozaoke.com${item.link}`;
const res = await got({
method: 'get',
url,
headers: {
Cookie: config.guozaoke.cookies,
},
});
const $ = load(res.data);
let content = $('div.ui-content').html();
content = content ? content.trim() : '';
const comments = $('.reply-item').map((i, el) => {
const $el = $(el);
const comment = $el.find('span.content').text().trim();
const author = $el.find('span.username').text();
return {
comment,
author,
};
});
if (comments && comments.length > 0) {
for (const item of comments) {
content += '<br>' + item.author + ': ' + item.comment;
const $ = load(res.data);
let content = $('div.ui-content').html();
content = content ? content.trim() : '';
const comments = $('.reply-item').map((i, el) => {
const $el = $(el);
const comment = $el.find('span.content').text().trim();
const author = $el.find('span.username').text();
return {
comment,
author,
};
});
if (comments && comments.length > 0) {
for (const item of comments) {
content += '<br>' + item.author + ': ' + item.comment;
}
}
}
item.description = content;
return item;
})
)) {
out.push(result);
}
item.description = content;
return item;
}),
{ concurrency: 2 }
);
return {
title: '过早客',
lib/routes/kcna/news.ts
+32 -32
View File
@@ -3,7 +3,7 @@ import { Route } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
import { art } from '@/utils/render';
import { fixDesc, fetchPhoto, fetchVideo } from './utils';
import path from 'node:path';
@@ -79,42 +79,42 @@ async function handler(ctx) {
// avoid being IP-banned
// if being banned, 103.35.255.254 (the last hop before www.kcna.kp - 175.45.176.71) will drop the packet
// verify that with `mtr www.kcna.kp -Tz`
const items = [];
for await (const item of asyncPool(3, list, (item) =>
cache.tryGet(item.link, async () => {
const response = await got(item.link);
const $ = load(response.data);
item.title = $('article-main-title').text() || item.title;
const items = await pMap(
list,
(item) =>
cache.tryGet(item.link, async () => {
const response = await got(item.link);
const $ = load(response.data);
item.title = $('article-main-title').text() || item.title;
const dateElem = $('.publish-time');
const dateString = dateElem.text().match(/\d+\.\d+\.\d+/);
dateElem.remove();
item.pubDate = dateString ? timezone(parseDate(dateString[0]), +9) : item.pubDate;
const dateElem = $('.publish-time');
const dateString = dateElem.text().match(/\d+\.\d+\.\d+/);
dateElem.remove();
item.pubDate = dateString ? timezone(parseDate(dateString[0]), +9) : item.pubDate;
const description = fixDesc($, $('.article-content-body .content-wrapper'));
const description = fixDesc($, $('.article-content-body .content-wrapper'));
// add picture and video
const media = $('.media-icon a')
.map((_, elem) => rootUrl + elem.attribs.href)
.get();
let photo, video;
await Promise.all(
media.map(async (medium) => {
if (medium.includes('/photo/')) {
photo = await fetchPhoto(ctx, medium);
} else if (medium.includes('/video/')) {
video = await fetchVideo(ctx, medium);
}
})
);
// add picture and video
const media = $('.media-icon a')
.map((_, elem) => rootUrl + elem.attribs.href)
.get();
let photo, video;
await Promise.all(
media.map(async (medium) => {
if (medium.includes('/photo/')) {
photo = await fetchPhoto(ctx, medium);
} else if (medium.includes('/video/')) {
video = await fetchVideo(ctx, medium);
}
})
);
item.description = art(path.join(__dirname, 'templates/news.art'), { description, photo, video });
item.description = art(path.join(__dirname, 'templates/news.art'), { description, photo, video });
return item;
})
)) {
items.push(item);
}
return item;
}),
{ concurrency: 3 }
);
return {
title,
lib/routes/luogu/contest.ts
+25 -25
View File
@@ -5,7 +5,7 @@ import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import MarkdownIt from 'markdown-it';
const md = MarkdownIt();
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
const baseUrl = 'https://www.luogu.com.cn';
@@ -64,31 +64,31 @@ async function handler() {
)
);
const result = [];
for await (const item of asyncPool(4, data.currentData.contests.result, (item) =>
cache.tryGet(`${baseUrl}/contest/${item.id}`, async () => {
const { data: response } = await got(`${baseUrl}/contest/${item.id}`);
const $ = load(response);
const data = JSON.parse(
decodeURIComponent(
$('script')
.text()
.match(/decodeURIComponent\("(.*)"\)/)[1]
)
);
const result = await pMap(
data.currentData.contests.result,
(item) =>
cache.tryGet(`${baseUrl}/contest/${item.id}`, async () => {
const { data: response } = await got(`${baseUrl}/contest/${item.id}`);
const $ = load(response);
const data = JSON.parse(
decodeURIComponent(
$('script')
.text()
.match(/decodeURIComponent\("(.*)"\)/)[1]
)
);
return {
title: item.name,
description: md.render(data.currentData.contest.description),
link: `${baseUrl}/contest/${item.id}`,
author: item.host.name,
pubDate: parseDate(item.startTime, 'X'),
category: [item.rated ? 'Rated' : null, typeMap.ruleType[item.ruleType], typeMap.visibilityType[item.visibilityType]].filter(Boolean),
};
})
)) {
result.push(item);
}
return {
title: item.name,
description: md.render(data.currentData.contest.description),
link: `${baseUrl}/contest/${item.id}`,
author: item.host.name,
pubDate: parseDate(item.startTime, 'X'),
category: [item.rated ? 'Rated' : null, typeMap.ruleType[item.ruleType], typeMap.visibilityType[item.visibilityType]].filter(Boolean),
};
}),
{ concurrency: 4 }
);
return {
title: $('head title').text(),
lib/routes/nextapple/realtime.ts
+28 -27
View File
@@ -3,7 +3,7 @@ import cache from '@/utils/cache';
import got from '@/utils/got';
import { parseDate } from '@/utils/parse-date';
import { load } from 'cheerio';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/realtime/:category?',
@@ -41,33 +41,34 @@ async function handler(ctx) {
const currentUrl = `https://tw.nextapple.com/realtime/${category}`;
const response = await got(currentUrl);
const $ = load(response.data);
const items = [];
for await (const item of asyncPool(5, $('article.infScroll'), (item) => {
const link = $(item).find('.post-title').attr('href');
return cache.tryGet(link, async () => {
const response = await got(link);
const $ = load(response.data);
const mainContent = $('#main-content');
const titleElement = mainContent.find('header h1');
const title = titleElement.text();
titleElement.remove();
const postMetaElement = mainContent.find('.post-meta');
const category = postMetaElement.find('.category').text();
const pubDate = parseDate(postMetaElement.find('time').attr('datetime'));
postMetaElement.remove();
$('.post-comments').remove();
const items = await pMap(
$('article.infScroll').toArray(),
(item) => {
const link = $(item).find('.post-title').attr('href');
return cache.tryGet(link, async () => {
const response = await got(link);
const $ = load(response.data);
const mainContent = $('#main-content');
const titleElement = mainContent.find('header h1');
const title = titleElement.text();
titleElement.remove();
const postMetaElement = mainContent.find('.post-meta');
const category = postMetaElement.find('.category').text();
const pubDate = parseDate(postMetaElement.find('time').attr('datetime'));
postMetaElement.remove();
$('.post-comments').remove();
return {
title,
description: mainContent.html(),
category,
pubDate,
link,
};
});
})) {
items.push(item);
}
return {
title,
description: mainContent.html(),
category,
pubDate,
link,
};
});
},
{ concurrency: 5 }
);
return {
title: $('title').text(),
lib/routes/shoppingdesign/posts.ts
+26 -25
View File
@@ -3,7 +3,7 @@ import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/posts',
@@ -34,31 +34,32 @@ async function handler() {
const currentUrl = 'https://www.shoppingdesign.com.tw/post?sn_f=1';
const response = await got(currentUrl);
const $ = load(response.data);
const items = [];
// maximum parallel requests on the target website are limited to 11.
for await (const data of asyncPool(10, $('article-item'), (item) => {
item = $(item);
const link = item.attr('url');
return cache.tryGet(link, async () => {
const response = await got(`${link}?sn_f=1`);
const $ = load(response.data);
const article = $('.left article .htmlview');
article.find('d-image').each(function () {
$(this).replaceWith(`<img src="${$(this).attr('lg')}">`);
});
const items = await pMap(
$('article-item').toArray(),
(item) => {
item = $(item);
const link = item.attr('url');
return cache.tryGet(link, async () => {
const response = await got(`${link}?sn_f=1`);
const $ = load(response.data);
const article = $('.left article .htmlview');
article.find('d-image').each(function () {
$(this).replaceWith(`<img src="${$(this).attr('lg')}">`);
});
return {
title: $('.left article .top_info h1').text(),
author: $('meta[name="my:author"]').attr('content'),
description: article.html(),
category: $('meta[name="my:category"]').attr('content'),
pubDate: parseDate($('meta[name="my:publish"]').attr('content')),
link,
};
});
})) {
items.push(data);
}
return {
title: $('.left article .top_info h1').text(),
author: $('meta[name="my:author"]').attr('content'),
description: article.html(),
category: $('meta[name="my:category"]').attr('content'),
pubDate: parseDate($('meta[name="my:publish"]').attr('content')),
link,
};
});
},
// maximum parallel requests on the target website are limited to 11.
{ concurrency: 10 }
);
return {
title: $('meta[property="og:title"]').attr('content'),
lib/routes/tfc-taiwan/utils.ts
+20 -25
View File
@@ -2,17 +2,9 @@ import got from '@/utils/got';
import { load } from 'cheerio';
import path from 'node:path';
import { art } from '@/utils/render';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
import { parseDate } from '@/utils/parse-date';
const asyncPoolAll = async (...args) => {
const results = [];
for await (const result of asyncPool(...args)) {
results.push(result);
}
return results;
};
const baseUrl = 'https://tfc-taiwan.org.tw';
const parseList = (item) => {
@@ -27,26 +19,29 @@ const parseList = (item) => {
};
const parseItems = (list, tryGet) =>
asyncPoolAll(10, list, (item) =>
tryGet(item.link, async () => {
const { data: response } = await got(item.link);
const $ = load(response);
pMap(
list,
(item) =>
tryGet(item.link, async () => {
const { data: response } = await got(item.link);
const $ = load(response);
$('.field-name-field-addthis, #fb-root, .fb-comments, .likecoin-embed, style[type="text/css"]').remove();
$('.field-name-field-addthis, #fb-root, .fb-comments, .likecoin-embed, style[type="text/css"]').remove();
item.description = art(path.join(__dirname, 'templates/article.art'), {
headerImage: item.image,
content: $('#block-system-main .node-content').html(),
});
item.description = art(path.join(__dirname, 'templates/article.art'), {
headerImage: item.image,
content: $('#block-system-main .node-content').html(),
});
item.pubDate = $('meta[property="article:published_time"]').attr('content');
item.updated = $('meta[property="article:modified_time"]').attr('content');
item.category = $('.node-tags .field-item')
.toArray()
.map((item) => $(item).text());
item.pubDate = $('meta[property="article:published_time"]').attr('content');
item.updated = $('meta[property="article:modified_time"]').attr('content');
item.category = $('.node-tags .field-item')
.toArray()
.map((item) => $(item).text());
return item;
})
return item;
}),
{ concurrency: 10 }
);
export { baseUrl, parseList, parseItems };
lib/routes/tradingview/blog.ts
+40 -40
View File
@@ -4,7 +4,7 @@ import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
import { art } from '@/utils/render';
import path from 'node:path';
@@ -26,7 +26,7 @@ async function handler(ctx) {
const $ = load(response);
const items = $('article[id]')
const list = $('article[id]')
.slice(0, limit)
.toArray()
.map((item) => {
@@ -55,48 +55,48 @@ async function handler(ctx) {
};
});
for await (const item of asyncPool(3, items, (item) =>
cache.tryGet(item.link, async () => {
const { data: detailResponse } = await got(item.link);
const items = await pMap(
list,
(item) =>
cache.tryGet(item.link, async () => {
const { data: detailResponse } = await got(item.link);
const content = load(detailResponse);
const content = load(detailResponse);
content('div.entry-content')
.find('img')
.each((_, e) => {
content(e).replaceWith(
art(path.join(__dirname, 'templates/description.art'), {
image: {
src: content(e)
.prop('src')
.replace(/-\d+x\d+\./, '.'),
width: content(e).prop('width'),
height: content(e).prop('height'),
},
})
);
content('div.entry-content')
.find('img')
.each((_, e) => {
content(e).replaceWith(
art(path.join(__dirname, 'templates/description.art'), {
image: {
src: content(e)
.prop('src')
.replace(/-\d+x\d+\./, '.'),
width: content(e).prop('width'),
height: content(e).prop('height'),
},
})
);
});
item.title = content('meta[property="og:title"]').prop('content');
item.description = art(path.join(__dirname, 'templates/description.art'), {
image: {
src: content('meta[property="og:image"]').prop('content'),
alt: item.title,
},
description: content('div.entry-content').html(),
});
item.author = content('meta[property="og:site_name"]').prop('content');
item.category = content('div.sections a.section')
.toArray()
.map((c) => content(c).text());
item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY');
item.title = content('meta[property="og:title"]').prop('content');
item.description = art(path.join(__dirname, 'templates/description.art'), {
image: {
src: content('meta[property="og:image"]').prop('content'),
alt: item.title,
},
description: content('div.entry-content').html(),
});
item.author = content('meta[property="og:site_name"]').prop('content');
item.category = content('div.sections a.section')
.toArray()
.map((c) => content(c).text());
item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY');
return item;
})
)) {
items.shift();
items.push(item);
}
return item;
}),
{ concurrency: 3 }
);
const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href;
lib/routes/wsj/news.ts
+3 -2
View File
@@ -1,7 +1,8 @@
import { Route } from '@/types';
import got from '@/utils/got';
import { load } from 'cheerio';
import { asyncPoolAll, parseArticle } from './utils';
import { parseArticle } from './utils';
import pMap from 'p-map';
const hostMap = {
'en-us': 'https://www.wsj.com',
'zh-cn': 'https://cn.wsj.com/zh-hans',
@@ -72,7 +73,7 @@ async function handler(ctx) {
item.test = key;
return item;
});
const items = await asyncPoolAll(10, list, (item) => parseArticle(item));
const items = await pMap(list, (item) => parseArticle(item), { concurrency: 10 });
return {
title: `WSJ${subTitle}`,
lib/routes/wsj/utils.ts
+1 -9
View File
@@ -1,5 +1,4 @@
import cache from '@/utils/cache';
import asyncPool from 'tiny-async-pool';
import { load } from 'cheerio';
import got from '@/utils/got';
import { parseDate } from '@/utils/parse-date';
@@ -111,11 +110,4 @@ const parseArticle = (item) =>
};
});
const asyncPoolAll = async (...args) => {
const results = [];
for await (const result of asyncPool(...args)) {
results.push(result);
}
return results;
};
export { asyncPoolAll, parseArticle };
export { parseArticle };
lib/routes/x-mol/paper.ts
+17 -22
View File
@@ -5,7 +5,7 @@ import { load } from 'cheerio';
import utils from './utils';
import { parseDate } from '@/utils/parse-date';
import timezone from '@/utils/timezone';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/paper/:type/:magazine',
@@ -57,31 +57,26 @@ async function handler(ctx) {
};
});
const asyncPoolAll = async (...args) => {
const results = [];
for await (const result of asyncPool(...args)) {
results.push(result);
}
return results;
};
const item = await pMap(
newsItem,
(element) =>
cache.tryGet(element.link, async () => {
const response = await got(element.link);
const $ = load(response.data);
const item = await asyncPoolAll(2, newsItem, (element) =>
cache.tryGet(element.link, async () => {
const response = await got(element.link);
const $ = load(response.data);
const description = $('.maga-content');
element.doi = description.find('.itsmblue').eq(1).text().trim();
const description = $('.maga-content');
element.doi = description.find('.itsmblue').eq(1).text().trim();
description.find('.itgaryfirst').remove();
description.find('span').eq(0).remove();
element.author = description.find('span').eq(0).text().trim();
description.find('span').eq(0).remove();
description.find('.itgaryfirst').remove();
description.find('span').eq(0).remove();
element.author = description.find('span').eq(0).text().trim();
description.find('span').eq(0).remove();
element.description = description.html();
element.description = description.html();
return element;
})
return element;
}),
{ concurrency: 2 }
);
return {
lib/routes/yamibo/bbs/forum.ts
+5 -4
View File
@@ -3,7 +3,8 @@ import type { Context } from 'hono';
import { config } from '@/config';
import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import { asyncPoolAll, fetchThread, generateDescription, getDate, bbsOrigin } from '../utils';
import { fetchThread, generateDescription, getDate, bbsOrigin } from '../utils';
import pMap from 'p-map';
import cache from '@/utils/cache';
export const route: Route = {
@@ -81,8 +82,7 @@ async function handler(ctx: Context): Promise<Data> {
};
});
items = await asyncPoolAll(
5,
items = await pMap(
items,
async (item) =>
(await cache.tryGet(item.link!, async () => {
@@ -105,7 +105,8 @@ async function handler(ctx: Context): Promise<Data> {
description,
pubDate: item.pubDate,
};
})) as DataItem
})) as DataItem,
{ concurrency: 5 }
);
return {
lib/routes/yamibo/utils.ts
-9
View File
@@ -2,7 +2,6 @@ import { parseDate } from '@/utils/parse-date';
import timezone from '@/utils/timezone';
import ofetch from '@/utils/ofetch';
import { config } from '@/config';
import asyncPool from 'tiny-async-pool';
import { JSDOM } from 'jsdom';
import type { Cheerio, Element } from 'cheerio';
@@ -104,11 +103,3 @@ export function generateDescription($item: Cheerio<Element>, postId: string) {
return description;
}
export async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
const results: Awaited<OUT[]> = [];
for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
results.push(result);
}
return results;
}
lib/routes/youtube/subscriptions.ts
+3 -11
View File
@@ -3,7 +3,7 @@ import cache from '@/utils/cache';
import { config } from '@/config';
import utils from './utils';
import { parseDate } from '@/utils/parse-date';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
import ConfigNotFoundError from '@/errors/types/config-not-found';
export const route: Route = {
@@ -51,18 +51,10 @@ async function handler(ctx) {
const channelIds = (await utils.getSubscriptions('snippet', cache)).data.items.map((item) => item.snippet.resourceId.channelId);
const playlistIds = [];
for await (const playlistId of asyncPool(30, channelIds, async (channelId) => (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads)) {
playlistIds.push(playlistId);
}
const playlistIds = await pMap(channelIds, async (channelId) => (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads, { concurrency: 30 });
let items = [];
for await (const item of asyncPool(30, playlistIds, async (playlistId) => (await utils.getPlaylistItems(playlistId, 'snippet', cache))?.data.items)) {
items.push(item);
}
let items = await pMap(playlistIds, async (playlistId) => (await utils.getPlaylistItems(playlistId, 'snippet', cache))?.data.items, { concurrency: 30 });
// https://measurethat.net/Benchmarks/Show/7223
// concat > reduce + concat >>> flat
items = items.flat();
items = items
lib/routes/zaker/channel.ts
+2 -5
View File
@@ -2,7 +2,7 @@ import { Route } from '@/types';
import cache from '@/utils/cache';
import * as cheerio from 'cheerio';
import { baseUrl, fetchItem, getSafeLineCookieWithData, parseList } from './utils';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
export const route: Route = {
path: '/channel/:id?',
@@ -31,10 +31,7 @@ async function handler(ctx) {
const feedTitle = $('head title').text();
const list = parseList($);
const items = [];
for await (const item of asyncPool(2, list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)))) {
items.push(item);
}
const items = await pMap(list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)), { concurrency: 2 });
return {
title: feedTitle,
lib/routes/zaker/focus.ts
+2 -5
View File
@@ -1,7 +1,7 @@
import { Route } from '@/types';
import cache from '@/utils/cache';
import * as cheerio from 'cheerio';
import asyncPool from 'tiny-async-pool';
import pMap from 'p-map';
import { baseUrl, fetchItem, getSafeLineCookieWithData, parseList } from './utils';
export const route: Route = {
@@ -26,10 +26,7 @@ async function handler() {
const $ = cheerio.load(data);
const list = parseList($);
const items = [];
for await (const item of asyncPool(2, list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)))) {
items.push(item);
}
const items = await pMap(list, (item) => cache.tryGet(item.link!, () => fetchItem(item, cookie)), { concurrency: 2 });
return {
title: 'ZAKER 精读新闻',
package.json
-2
View File
@@ -125,7 +125,6 @@
"socks-proxy-agent": "8.0.5",
"source-map": "0.7.4",
"telegram": "2.26.22",
"tiny-async-pool": "2.1.0",
"title": "4.0.1",
"tldts": "7.0.2",
"tosource": "2.0.0-alpha.3",
@@ -166,7 +165,6 @@
"@types/node": "22.14.1",
"@types/sanitize-html": "2.15.0",
"@types/supertest": "6.0.3",
"@types/tiny-async-pool": "2.0.3",
"@types/title": "3.4.3",
"@types/uuid": "10.0.0",
"@typescript-eslint/eslint-plugin": "8.31.0",
pnpm-lock.yaml
Generated
-16
View File
@@ -242,9 +242,6 @@ importers:
telegram:
specifier: 2.26.22
version: 2.26.22
tiny-async-pool:
specifier: 2.1.0
version: 2.1.0
title:
specifier: 4.0.1
version: 4.0.1
@@ -360,9 +357,6 @@ importers:
'@types/supertest':
specifier: 6.0.3
version: 6.0.3
'@types/tiny-async-pool':
specifier: 2.0.3
version: 2.0.3
'@types/title':
specifier: 3.4.3
version: 3.4.3
@@ -2534,9 +2528,6 @@ packages:
'@types/tedious@4.0.14':
resolution: {integrity: sha512-KHPsfX/FoVbUGbyYvk1q9MMQHLPeRZhRJZdO45Q4YjvFkv4hMNghCWTvy7rdKessBsmtz4euWCWAB6/tVpI1Iw==}
'@types/tiny-async-pool@2.0.3':
resolution: {integrity: sha512-n3l1s538tKo9RBoHs4I3DG/VmD3VYhF5mHcgu1sU4Lq7JCNBtxnpBy3OkWSbZsp5r5QOuplh2UkXXXwufoAuNQ==}
'@types/title@3.4.3':
resolution: {integrity: sha512-mjupLOb4kwUuoUFokkacy/VMRVBH2qtqZ5AX7K7iha6+iKIkX80n/Y4EoNVEVRmer8dYJU/ry+fppUaDFVQh7Q==}
@@ -5881,9 +5872,6 @@ packages:
through@2.3.8:
resolution: {integrity: sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==}
tiny-async-pool@2.1.0:
resolution: {integrity: sha512-ltAHPh/9k0STRQqaoUX52NH4ZQYAJz24ZAEwf1Zm+HYg3l9OXTWeqWKyYsHu40wF/F0rxd2N2bk5sLvX2qlSvg==}
tinybench@2.9.0:
resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==}
@@ -8646,8 +8634,6 @@ snapshots:
dependencies:
'@types/node': 22.14.1
'@types/tiny-async-pool@2.0.3': {}
'@types/title@3.4.3': {}
'@types/tough-cookie@4.0.5': {}
@@ -12483,8 +12469,6 @@ snapshots:
through@2.3.8: {}
tiny-async-pool@2.1.0: {}
tinybench@2.9.0: {}
tinyexec@0.3.2: {}