From 9b6594bbad0dd798ef0de68d411bf8a6d126b363 Mon Sep 17 00:00:00 2001 From: StevenRCE0 <40051361+StevenRCE0@users.noreply.github.com> Date: Thu, 16 Mar 2023 17:39:08 +0800 Subject: [PATCH] feat(route): Update route OpenAI (#12113) * Updated route OpenAI * style: auto format * Improved route OpenAI * Use cheerio to remove comments * style: auto format * Update lib/v2/openai/radar.js --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/en/new-media.md | 8 +-- docs/new-media.md | 8 +-- lib/router.js | 3 - lib/routes/openai/blog.js | 52 ---------------- lib/v2/openai/blog.js | 97 +++++++++++++++++++++++++++++ lib/v2/openai/maintainer.js | 3 + lib/v2/openai/radar.js | 19 ++++++ lib/v2/openai/router.js | 3 + lib/v2/openai/templates/article.art | 2 + 9 files changed, 132 insertions(+), 63 deletions(-) delete mode 100644 lib/routes/openai/blog.js create mode 100644 lib/v2/openai/blog.js create mode 100644 lib/v2/openai/maintainer.js create mode 100644 lib/v2/openai/radar.js create mode 100644 lib/v2/openai/router.js create mode 100644 lib/v2/openai/templates/article.art diff --git a/docs/en/new-media.md b/docs/en/new-media.md index e8cd57ac11..3b49206e59 100644 --- a/docs/en/new-media.md +++ b/docs/en/new-media.md @@ -554,11 +554,11 @@ This route provides a flexible plan with full text content to subscribe specific ### Blog - + -| All | Research | Announcements | Events | Milestones | -| --- | -------- | ------------- | ------ | ---------- | -| | research | announcements | events | milestones | +| All | Announcements | Events | Safety & Alignment | Community | Product | Culture & Careers | Milestones | Research | +| --- | ------------- | ------ | ------------------ | --------- | ------- | ------------------- | ---------- | -------- | +| | announcements | events | safety-alignment | community | product | culture-and-careers | milestones | research | diff --git a/docs/new-media.md b/docs/new-media.md index 92a3bb0063..a1d109e809 100644 --- a/docs/new-media.md +++ b/docs/new-media.md @@ -1024,11 +1024,11 @@ IPFS 网关有可能失效，那时候换成其他网关。 ### Blog - + -| All | Research | Announcements | Events | Milestones | -| --- | -------- | ------------- | ------ | ---------- | -| | research | announcements | events | milestones | +| All | Announcements | Events | Safety & Alignment | Community | Product | Culture & Careers | Milestones | Research | +| --- | ------------- | ------ | ------------------ | --------- | ------- | ------------------- | ---------- | -------- | +| | announcements | events | safety-alignment | community | product | culture-and-careers | milestones | research | diff --git a/lib/router.js b/lib/router.js index 4c7705f35b..f598e63e7a 100644 --- a/lib/router.js +++ b/lib/router.js @@ -3419,9 +3419,6 @@ router.get('/blizzard/news/:language?/:category?', lazyloadRouteHandler('./route // DeepL router.get('/deepl/blog/:lang?', lazyloadRouteHandler('./routes/deepl/blog')); -// OpenAI -router.get('/openai/blog/:tag?', lazyloadRouteHandler('./routes/openai/blog')); - // 小木虫 router.get('/muchong/journal/:type?', lazyloadRouteHandler('./routes/muchong/journal')); router.get('/muchong/:id/:type?/:sort?', lazyloadRouteHandler('./routes/muchong/index')); diff --git a/lib/routes/openai/blog.js b/lib/routes/openai/blog.js deleted file mode 100644 index 4d0abfbf84..0000000000 --- a/lib/routes/openai/blog.js +++ /dev/null @@ -1,52 +0,0 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); - -module.exports = async (ctx) => { - const tag = ctx.params.tag || ''; - - const rootUrl = 'https://openai.com'; - const currentUrl = `${rootUrl}/blog${tag === '' ? '' : `/tags/${tag}`}`; - const response = await got({ - method: 'get', - url: currentUrl, - }); - - const $ = cheerio.load(response.data); - - const list = $('.col-12 a') - .slice(0, 5) - .map((_, item) => { - item = $(item); - return { - link: `${rootUrl}${item.attr('href')}`, - }; - }) - .get(); - - const items = await Promise.all( - list.map((item) => - ctx.cache.tryGet(item.link, async () => { - const detailResponse = await got({ - method: 'get', - url: item.link, - }); - const content = cheerio.load(detailResponse.data); - - item.title = content('title').text(); - - content('.aside').remove(); - - item.description = content('.content').html(); - item.pubDate = new Date(content('time').attr('datetime')).toUTCString(); - - return item; - }) - ) - ); - - ctx.state.data = { - title: $('title').text(), - link: currentUrl, - item: items, - }; -}; diff --git a/lib/v2/openai/blog.js b/lib/v2/openai/blog.js new file mode 100644 index 0000000000..dabc4dd545 --- /dev/null +++ b/lib/v2/openai/blog.js @@ -0,0 +1,97 @@ +const got = require('@/utils/got'); +const cheerio = require('cheerio'); +const { art } = require('@/utils/render'); +const { toTitleCase } = require('@/utils/common-utils'); +const path = require('path'); + +module.exports = async (ctx) => { + const tag = ctx.params.tag || ''; + + const rootUrl = 'https://openai.com'; + const blogRootUrl = 'https://openai.com/blog'; + const blogOriginUrl = `${rootUrl}/blog${tag === '' ? '' : `?topics=${tag}`}`; + + // Find API base URL + const initResponse = await got({ + method: 'get', + url: blogRootUrl, + }); + + let apiBaseUrl = initResponse.data + .toString() + .match(/(?<=TWILL_API_BASE:").+?(?=")/)[0] + .replaceAll('\\u002F', '/'); + apiBaseUrl = apiBaseUrl + '/api/v1/blog-details'; + const apiUrl = new URL(apiBaseUrl); + + // Construct API query + apiUrl.searchParams.append('sort', '-publicationDate,-createdAt'); + apiUrl.searchParams.append('page[size]', '20'); + apiUrl.searchParams.append('page[number]', '1'); + apiUrl.searchParams.append('include', 'media,topics,authors'); + if (tag) { + apiUrl.searchParams.append('filter[topics][slugs][0]', tag); + } + + const response = await got({ + method: 'get', + url: apiUrl, + }); + + const list = response.data.data.filter((entry) => entry.type === 'blog-details'); + + const items = await Promise.all( + list.map((item) => { + const attributes = item.attributes; + const textUrl = `${blogRootUrl}/${attributes.slug}`; + return ctx.cache.tryGet(attributes.slug, async () => { + const detailResponse = await got({ + method: 'get', + url: textUrl, + }); + let content = cheerio.load(detailResponse.data); + + const authors = content('[aria-labelledby="metaAuthorsHeading"] > li > a > span > span') + .toArray() + .map((entry) => content(entry).text()) + .join(', '); + + // Leave out comments + const comments = content('*') + .contents() + .filter(function () { + return this.nodeType === 8; + }); + comments.remove(); + + content = content('#content'); + + const imageSrc = attributes.seo.ogImageSrc; + const imageAlt = attributes.seo.ogImageAlt; + + const article = art(path.join(__dirname, 'templates/article.art'), { + content, + imageSrc, + imageAlt, + }); + + return { + title: attributes.title, + author: authors, + description: article, + pubDate: attributes.createdAt, + category: attributes.tags.map((tag) => tag.title), + link: textUrl, + }; + }); + }) + ); + + const title = `OpenAI Blog${tag ? ` - ${toTitleCase(tag)}` : ''}`; + + ctx.state.data = { + title, + link: blogOriginUrl, + item: items, + }; +}; diff --git a/lib/v2/openai/maintainer.js b/lib/v2/openai/maintainer.js new file mode 100644 index 0000000000..65fa1d8d2b --- /dev/null +++ b/lib/v2/openai/maintainer.js @@ -0,0 +1,3 @@ +module.exports = { + '/blog/:tag?': ['StevenRCE0', 'nczitzk'], +}; diff --git a/lib/v2/openai/radar.js b/lib/v2/openai/radar.js new file mode 100644 index 0000000000..7ee74e532d --- /dev/null +++ b/lib/v2/openai/radar.js @@ -0,0 +1,19 @@ +module.exports = { + 'openai.com': { + _name: 'OpenAI', + '.': [ + { + title: 'Blog', + docs: 'https://docs.rsshub.app/en/new-media.html#openai', + source: '/blog', + target: (_, url) => { + const topics = new URL(url).searchParams.get('topics'); + if (topics) { + return `/openai/blog/${topics}`; + } + return '/openai/blog'; + }, + }, + ], + }, +}; diff --git a/lib/v2/openai/router.js b/lib/v2/openai/router.js new file mode 100644 index 0000000000..5b178ef0be --- /dev/null +++ b/lib/v2/openai/router.js @@ -0,0 +1,3 @@ +module.exports = function (router) { + router.get('/blog/:tag?', require('./blog')); +}; diff --git a/lib/v2/openai/templates/article.art b/lib/v2/openai/templates/article.art new file mode 100644 index 0000000000..3d661adfbe --- /dev/null +++ b/lib/v2/openai/templates/article.art @@ -0,0 +1,2 @@ +{{ imageAlt }} +{{@ content }}