fix(route/nea): fetch ghs items logic

This commit is contained in:
pseudoyu
2025-04-15 17:01:46 +08:00
parent f17f16bbc4
commit 87a679e265
2 changed files with 66 additions and 39 deletions
+1
View File
@@ -32,6 +32,7 @@ yarn.lock
yarn-error.log
# ai ide rules
memory-bank/
.roomodes
.cursorrules
.windsurfrules
+65 -39
View File
@@ -1,9 +1,10 @@
import { Route } from '@/types';
import type { Route, DataItem } from '@/types';
import cache from '@/utils/cache';
import got from '@/utils/got';
import ofetch from '@/utils/ofetch';
import { load } from 'cheerio';
import timezone from '@/utils/timezone';
import { parseDate } from '@/utils/parse-date';
import { URL } from 'url';
import timezone from '@/utils/timezone';
export const route: Route = {
path: '/nea/sjzz/ghs',
@@ -21,64 +22,89 @@ export const route: Route = {
radar: [
{
source: ['nea.gov.cn/sjzz/ghs/'],
target: '/nea/sjzz/ghs',
},
],
name: '发展规划司',
maintainers: ['nczitzk'],
maintainers: ['nczitzk', 'pseudoyu'],
handler,
url: 'nea.gov.cn/sjzz/ghs/',
url: 'www.nea.gov.cn/sjzz/ghs/',
};
async function handler(ctx) {
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 35;
const rootUrl = 'https://www.nea.gov.cn';
const currentUrl = new URL('sjzz/ghs/', rootUrl).href;
const jsonUrl = new URL('sjzz/ghs/ds_99a01b29f9a24ab58f7d64b36489500e.json', rootUrl).href;
const { data: response } = await got(currentUrl);
const jsonData: NeaGhsResponse = await ofetch(jsonUrl);
const $ = load(response);
const list: InternalDataItem[] = jsonData.datasource.slice(0, limit).map((item) => {
const itemLink = new URL(item.publishUrl, rootUrl).href;
let items = $('div.right_box ul li')
.slice(0, limit)
.toArray()
.map((item) => {
item = $(item);
const $title = load(item.showTitle);
const titleText = $title.text();
const a = item.find('a');
return {
title: titleText,
link: itemLink,
pubDate: parseDate(item.publishTime),
description: item.summary?.trim() || titleText,
author: item.sourceText?.trim() || undefined,
category: [] as string[],
};
});
return {
title: a.text(),
link: a.prop('href'),
pubDate: parseDate(item.find('span.date-tex').text()),
};
});
items = await Promise.all(
items.map((item) =>
const items = await Promise.all(
list.map((item: InternalDataItem) =>
cache.tryGet(item.link, async () => {
const { data: detailResponse } = await got(item.link);
const content = load(detailResponse);
item.title = content('meta[name="ArticleTitle"]').prop('content');
item.description = content('td.detail').html() || content('div.article-content td').html();
item.author = content('meta[name="ContentSource"]').prop('content');
item.category = content('meta[name="keywords"]').prop('content').split(/,/);
item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').prop('content')), +8);
try {
const detailResponse = await ofetch(item.link);
const content = load(detailResponse);
item.title = content('meta[name="ArticleTitle"]').prop('content') || item.title;
item.description = content('td.detail').html() || content('div.article-content td').html() || item.description;
item.author = content('meta[name="ContentSource"]').prop('content') || item.author;
item.category = content('meta[name="keywords"]').prop('content')?.split(/,/) ?? item.category;
const detailPubDate = content('meta[name="PubDate"]').prop('content');
item.pubDate = detailPubDate ? timezone(parseDate(detailPubDate), +8) : item.pubDate;
} catch {
// logger.error(`Failed to fetch detail for ${item.link}`);
}
return item;
})
)
);
const filteredItems: DataItem[] = items.filter(Boolean) as DataItem[];
return {
item: items,
title: $('title').text(),
link: currentUrl,
description: $('meta[name="ColumnDescription"]').prop('content'),
language: 'zh',
subtitle: $('meta[name="ColumnType"]').prop('content'),
author: $('meta[name="ColumnKeywords"]').prop('content'),
item: filteredItems,
title: '国家能源局 - 发展规划司工作进展',
link: 'https://www.nea.gov.cn/sjzz/ghs/',
description: '国家能源局 - 发展规划司工作进展',
};
}
interface NeaGhsItem {
showTitle: string;
publishUrl: string;
publishTime: string;
summary?: string;
sourceText?: string;
}
interface NeaGhsResponse {
categoryName?: string;
categoryDesc?: string;
datasource: NeaGhsItem[];
}
interface InternalDataItem {
title: string;
link: string;
pubDate: Date;
description: string;
author?: string;
category: string[];
}