diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml
new file mode 100644
index 0000000000..f3ef22e85f
--- /dev/null
+++ b/.github/workflows/scrape.yml
@@ -0,0 +1,29 @@
+name: Update meilisearch index
+
+on:
+ push:
+ branches:
+ - master
+ paths:
+ - '.github/workflows/scrape.yml'
+ - 'docs/**'
+
+jobs:
+ scrape-docs:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Wait for Netlify to finish
+ run: sleep 3m
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Run docs-scraper
+ env:
+ HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }}
+ API_KEY: ${{ secrets.MEILISEARCH_API_KEY }}
+ CONFIG_FILE_PATH: ${{ github.workspace }}/scripts/docs-scraper/docs.rsshub.app.json
+ run: |
+ docker run -t --rm \
+ -e MEILISEARCH_HOST_URL=$HOST_URL \
+ -e MEILISEARCH_API_KEY=$API_KEY \
+ -v $CONFIG_FILE_PATH:/docs-scraper/config.json \
+ getmeili/docs-scraper pipenv run ./docs_scraper config.json
diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js
index 76467d5f7e..a29df462fe 100644
--- a/docs/.vuepress/config.js
+++ b/docs/.vuepress/config.js
@@ -20,6 +20,15 @@ module.exports = {
},
},
'@vuepress/back-to-top': true,
+ sitemap: {
+ hostname: 'https://docs.rsshub.app',
+ },
+ 'vuepress-plugin-meilisearch': {
+ hostUrl: 'https://meilisearch.rsshub.app',
+ apiKey: '375c36cd9573a2c1d1e536214158c37120fdd0ba6cd8829f7a848e940cc22245',
+ indexUid: 'rsshub',
+ maxSuggestions: 14,
+ },
},
locales: {
'/': {
@@ -73,14 +82,6 @@ module.exports = {
editLinks: true,
docsDir: 'docs',
smoothScroll: true,
- algolia: {
- apiKey: '4c5f9a5a2c10e6bec73c0f07dbe356a6',
- indexName: 'rsshub',
- appId: 'I63CFUQ7XF',
- algoliaOptions: {
- hitsPerPage: 14,
- },
- },
locales: {
'/': {
lang: 'zh-CN',
diff --git a/docs/government.md b/docs/government.md
index 5231109df7..6675a51899 100644
--- a/docs/government.md
+++ b/docs/government.md
@@ -1247,7 +1247,7 @@ pageClass: routes
### 政务公开
-{{ content }}
+<% } %>
+{{ /if }}
\ No newline at end of file
diff --git a/lib/v2/iresearch/weekly.js b/lib/v2/iresearch/weekly.js
new file mode 100644
index 0000000000..f0d516b4a0
--- /dev/null
+++ b/lib/v2/iresearch/weekly.js
@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+ const category = ctx.params.category ?? '';
+
+ const rootUrl = 'https://www.iresearch.com.cn';
+ const currentUrl = `${rootUrl}/report.shtml?type=4`;
+ const apiUrl = `${rootUrl}/api/json/report/ireport.json`;
+
+ const response = await got({
+ method: 'get',
+ url: apiUrl,
+ });
+
+ const items = JSON.parse(response.data.slice(1))
+ .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 200)
+ .filter((item) => (category ? item.classname === category : true))
+ .map((item) => ({
+ title: item.reportname,
+ pubDate: parseDate(item.addtime),
+ link: `${rootUrl}/report/detail?id=${item.id}`,
+ category: [item.classname].concat(item.keywords.split(',')),
+ description: art(path.join(__dirname, 'templates/weekly.art'), {
+ id: item.id,
+ cover: item.reportpic,
+ content: item.shortcoutent,
+ pages: item.PagesCount,
+ }),
+ }));
+
+ ctx.state.data = {
+ title: '艾瑞咨询 - 周度市场观察',
+ link: currentUrl,
+ item: items,
+ };
+};
diff --git a/lib/v2/stheadline/maintainer.js b/lib/v2/stheadline/maintainer.js
new file mode 100644
index 0000000000..46206e7f34
--- /dev/null
+++ b/lib/v2/stheadline/maintainer.js
@@ -0,0 +1,3 @@
+module.exports = {
+ '/std/realtime/:category*': ['TonyRL'],
+};
diff --git a/lib/v2/stheadline/radar.js b/lib/v2/stheadline/radar.js
new file mode 100644
index 0000000000..bfdc2be3b1
--- /dev/null
+++ b/lib/v2/stheadline/radar.js
@@ -0,0 +1,13 @@
+module.exports = {
+ 'stheadline.com': {
+ _name: '星島日報',
+ std: [
+ {
+ title: '即時',
+ docs: 'https://docs.rsshub.app/traditional-media.html#xing-dao-ri-bao',
+ source: ['/realtime/*category'],
+ target: (params) => `/stdheadline/std/realtime/${params.category}`,
+ },
+ ],
+ },
+};
diff --git a/lib/v2/stheadline/router.js b/lib/v2/stheadline/router.js
new file mode 100644
index 0000000000..ee345be8b5
--- /dev/null
+++ b/lib/v2/stheadline/router.js
@@ -0,0 +1,3 @@
+module.exports = (router) => {
+ router.get('/std/realtime/:category*', require('./std/realtime'));
+};
diff --git a/lib/v2/stheadline/std/realtime.js b/lib/v2/stheadline/std/realtime.js
new file mode 100644
index 0000000000..2fea18068b
--- /dev/null
+++ b/lib/v2/stheadline/std/realtime.js
@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const baseUrl = 'https://std.stheadline.com';
+
+module.exports = async (ctx) => {
+ const { category = '即時' } = ctx.params;
+ const url = `${baseUrl}/realtime/${category}`;
+ const { data: response } = await got(url);
+ const $ = cheerio.load(response);
+
+ const items = $('.col-md-9 .media-body .h5 a')
+ .toArray()
+ .map((item) => {
+ item = $(item);
+ return {
+ title: item.attr('title'),
+ link: item.attr('href'),
+ guid: item.attr('href').substring(0, item.attr('href').lastIndexOf('/')),
+ };
+ });
+
+ await Promise.all(
+ items.map((item) =>
+ ctx.cache.tryGet(item.link, async () => {
+ const { data: response } = await got(item.link);
+ const $ = cheerio.load(response);
+
+ item.description = $('.paragraphs').html();
+ item.pubDate = timezone(parseDate($('.content .date').text()));
+
+ return item;
+ })
+ )
+ );
+
+ ctx.state.data = {
+ title: $('head title').text(),
+ description: $('meta[name=description]').attr('content'),
+ image: 'https://std.stheadline.com/dist/images/favicon/icon-512.png',
+ link: url,
+ item: items,
+ };
+};
diff --git a/lib/v2/trow/portal.js b/lib/v2/trow/portal.js
index 0e0b6a6254..b2cc303c6b 100644
--- a/lib/v2/trow/portal.js
+++ b/lib/v2/trow/portal.js
@@ -4,11 +4,21 @@ const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
module.exports = async (ctx) => {
- const response = await got({
- method: 'get',
+ let data;
+ const response = await got.extend({ followRedirect: false }).get({
url: `https://trow.cc`,
});
- const data = response.data;
+ if (response.statusCode === 302) {
+ const response2 = await got.extend({ followRedirect: false }).get({
+ url: `https://trow.cc`,
+ headers: {
+ cookie: response.headers['set-cookie'],
+ },
+ });
+ data = response2.data;
+ } else {
+ data = response.data;
+ }
const $ = cheerio.load(data);
const list = $('#portal_content .borderwrap[style="display:show"]');
diff --git a/lib/v2/xjtu/dyyy/index.js b/lib/v2/xjtu/dyyy/index.js
new file mode 100644
index 0000000000..629720d623
--- /dev/null
+++ b/lib/v2/xjtu/dyyy/index.js
@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'http://www.dyyy.xjtu.edu.cn';
+
+module.exports = async (ctx) => {
+ const { path } = ctx.params;
+ const response = await got(`${baseUrl}/${path}.htm`);
+
+ const $ = cheerio.load(response.data);
+
+ const items = $('.list_right_con div li')
+ .toArray()
+ .map((item) => {
+ item = $(item);
+ return {
+ title: item.find('a').attr('title'),
+ link: new URL(item.find('a').attr('href'), response.url).href,
+ pubDate: parseDate(item.find('.data').text()),
+ };
+ });
+
+ await Promise.all(
+ items.map((item) =>
+ ctx.cache.tryGet(item.link, async () => {
+ const { data: response } = await got(item.link);
+ const $ = cheerio.load(response);
+
+ item.author = $('.content_source')
+ .text()
+ .match(/责任编辑:(.*)\(点击/)[1];
+ item.description = $('.content_con').html();
+ return item;
+ })
+ )
+ );
+
+ ctx.state.data = {
+ title: $('head title').text(),
+ link: `${baseUrl}/${path}.htm`,
+ item: items,
+ };
+};
diff --git a/lib/v2/xjtu/maintainer.js b/lib/v2/xjtu/maintainer.js
index 9e81888f06..bdd4b9f039 100644
--- a/lib/v2/xjtu/maintainer.js
+++ b/lib/v2/xjtu/maintainer.js
@@ -1,6 +1,7 @@
module.exports = {
'/2yuan/news/:id?': ['nczitzk'],
'/dean/:subpath+': ['hoilc'],
+ '/dyyy/:path+': ['TonyRL'],
'/ee/:id?': ['DylanXie123'],
'/gs/tzgg': ['nczitzk'],
'/international/:subpath+': ['guitaoliu'],
diff --git a/lib/v2/xjtu/radar.js b/lib/v2/xjtu/radar.js
index 7594dfaac2..bace2feb6c 100644
--- a/lib/v2/xjtu/radar.js
+++ b/lib/v2/xjtu/radar.js
@@ -49,5 +49,13 @@ module.exports = {
target: (params, url) => `/xjtu/std/${new URL(url).toString().match(/\/(\w+)\.htm/)[1]}`,
},
],
+ 'www.dyyy': [
+ {
+ title: '第一附属医院新闻',
+ docs: 'https://docs.rsshub.app/university.html#xi-an-jiao-tong-da-xue',
+ source: ['/*'],
+ target: (_, url) => `/xjtu/dyyy${new URL(url).pathname.replace('.htm', '')}`,
+ },
+ ],
},
};
diff --git a/lib/v2/xjtu/router.js b/lib/v2/xjtu/router.js
index 77640e936a..fcf1189718 100644
--- a/lib/v2/xjtu/router.js
+++ b/lib/v2/xjtu/router.js
@@ -1,6 +1,7 @@
module.exports = function (router) {
router.get('/2yuan/news/:id?', require('./2yuan/news'));
router.get('/dean/:subpath+', require('./dean'));
+ router.get('/dyyy/:path+', require('./dyyy/index'));
router.get('/ee/:id?', require('./ee'));
router.get('/gs/tzgg', require('./gs/tzgg'));
router.get('/international/:subpath+', require('./international'));
diff --git a/package.json b/package.json
index 98a93f6938..06fed2bbc1 100644
--- a/package.json
+++ b/package.json
@@ -74,6 +74,8 @@
"tosource": "2.0.0-alpha.3",
"unified": "9.2.2",
"vuepress": "1.9.7",
+ "vuepress-plugin-meilisearch": "0.12.2",
+ "vuepress-plugin-sitemap": "2.3.1",
"yorkie": "2.0.0"
},
"dependencies": {
diff --git a/scripts/docs-scraper/docs.rsshub.app.json b/scripts/docs-scraper/docs.rsshub.app.json
new file mode 100644
index 0000000000..7e67509aa3
--- /dev/null
+++ b/scripts/docs-scraper/docs.rsshub.app.json
@@ -0,0 +1,21 @@
+{
+ "index_uid": "rsshub",
+ "sitemap_urls": ["https://docs.rsshub.app/sitemap.xml"],
+ "start_urls": ["https://docs.rsshub.app"],
+ "selectors": {
+ "lvl0": {
+ "selector": ".sidebar-heading.open",
+ "global": true,
+ "default_value": "路由"
+ },
+ "lvl1": ".theme-default-content h1",
+ "lvl2": ".theme-default-content h2",
+ "lvl3": ".theme-default-content h3",
+ "lvl4": ".theme-default-content h4",
+ "lvl5": ".theme-default-content h5",
+ "text": ".theme-default-content p, .docs-content li"
+ },
+ "strip_chars": " .,;:#",
+ "scrap_start_urls": true,
+ "min_indexed_level": 1
+}
diff --git a/yarn.lock b/yarn.lock
index eafedc3e09..964dccb8af 100755
--- a/yarn.lock
+++ b/yarn.lock
@@ -3007,6 +3007,13 @@ autocomplete.js@0.36.0:
dependencies:
immediate "^3.2.3"
+autocomplete.js@^0.38.1:
+ version "0.38.1"
+ resolved "https://registry.yarnpkg.com/autocomplete.js/-/autocomplete.js-0.38.1.tgz#9b006c985d996165ebbc62af33f5b4c32d209cc2"
+ integrity sha512-6pSJzuRMY3pqpozt+SXThl2DmJfma8Bi3SVFbZHS0PW/N72bOUv+Db0jAh2cWOhTsA4X+GNmKvIl8wExJTnN9w==
+ dependencies:
+ immediate "^3.2.3"
+
autoprefixer@^9.5.1:
version "9.8.8"
resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-9.8.8.tgz#fd4bd4595385fa6f06599de749a4d5f7a474957a"
@@ -4391,7 +4398,7 @@ cross-env@7.0.3:
dependencies:
cross-spawn "^7.0.1"
-cross-fetch@3.1.5:
+cross-fetch@3.1.5, cross-fetch@^3.1.5:
version "3.1.5"
resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.5.tgz#e1389f44d9e7ba767907f7af8454787952ab534f"
integrity sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==
@@ -5044,6 +5051,16 @@ dns-txt@^2.0.2:
dependencies:
buffer-indexof "^1.0.0"
+docs-searchbar.js@^2.1.1:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/docs-searchbar.js/-/docs-searchbar.js-2.2.0.tgz#e60eb0ee4f67853c41fdf7d3a4ef6ac7ac2c8027"
+ integrity sha512-rleKMiTkxG14M+6yt3+H8H0w/GKxWsNMinf4+zTVz9aVi04oJ8u37kktTiW3qkpPl9VQ7/i7Yv9wykukMqVxfw==
+ dependencies:
+ autocomplete.js "^0.38.1"
+ meilisearch "^0.27.0"
+ to-factory "^1.0.0"
+ zepto "^1.2.0"
+
docsearch.js@^2.5.2:
version "2.6.3"
resolved "https://registry.yarnpkg.com/docsearch.js/-/docsearch.js-2.6.3.tgz#57cb4600d3b6553c677e7cbbe6a734593e38625d"
@@ -9111,6 +9128,11 @@ lodash.bind@^4.1.4:
resolved "https://registry.yarnpkg.com/lodash.bind/-/lodash.bind-4.2.1.tgz#7ae3017e939622ac31b7d7d7dcb1b34db1690d35"
integrity sha512-lxdsn7xxlCymgLYo1gGvVrfHmkjDiyqVv62FAeF2i5ta72BipE1SLxw8hPEPLhD4/247Ijw07UQH7Hq/chT5LA==
+lodash.chunk@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/lodash.chunk/-/lodash.chunk-4.2.0.tgz#66e5ce1f76ed27b4303d8c6512e8d1216e8106bc"
+ integrity sha512-ZzydJKfUHJwHa+hF5X66zLFCBrWn5GeF28OHEr4WVWtNDXlQ/IjWKPBiikqKo2ne0+v6JgCgJ0GzJp8k8bHC7w==
+
lodash.clonedeep@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
@@ -9166,6 +9188,11 @@ lodash.merge@^4.4.0, lodash.merge@^4.6.2:
resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a"
integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==
+lodash.padstart@^4.6.1:
+ version "4.6.1"
+ resolved "https://registry.yarnpkg.com/lodash.padstart/-/lodash.padstart-4.6.1.tgz#d2e3eebff0d9d39ad50f5cbd1b52a7bce6bb611b"
+ integrity sha512-sW73O6S8+Tg66eY56DBk85aQzzUJDtpoXFBgELMd5P/SotAguo+1kYO6RuYgXxA4HJH3LFTFPASX6ET6bjfriw==
+
lodash.pick@^4.2.1:
version "4.4.0"
resolved "https://registry.yarnpkg.com/lodash.pick/-/lodash.pick-4.4.0.tgz#52f05610fff9ded422611441ed1fc123a03001b3"
@@ -9558,6 +9585,13 @@ media-typer@0.3.0:
resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==
+meilisearch@^0.27.0:
+ version "0.27.0"
+ resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.27.0.tgz#8bd57ddb77b975f93e054cb977b951c488ece297"
+ integrity sha512-kZOZFIuSO7c6xRf+Y2/9/h6A9pl0sCl/G44X4KuaSwxGbruOZPhmxbeVEgLHBv4pUFvQ56rNVTA/2d/5GCU1YA==
+ dependencies:
+ cross-fetch "^3.1.5"
+
memory-fs@^0.4.1:
version "0.4.1"
resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552"
@@ -12555,6 +12589,16 @@ sisteransi@^1.0.5:
resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.5.tgz#134d681297756437cc05ca01370d3a7a571075ed"
integrity sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==
+sitemap@^3.0.0:
+ version "3.2.2"
+ resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-3.2.2.tgz#3f77c358fa97b555c879e457098e39910095c62b"
+ integrity sha512-TModL/WU4m2q/mQcrDgNANn0P4LwprM9MMvG4hu5zP4c6IIKs2YLTu6nXXnNr8ODW/WFtxKggiJ1EGn2W0GNmg==
+ dependencies:
+ lodash.chunk "^4.2.0"
+ lodash.padstart "^4.6.1"
+ whatwg-url "^7.0.0"
+ xmlbuilder "^13.0.0"
+
slash@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"
@@ -14145,6 +14189,20 @@ vuepress-plugin-container@^2.0.2:
"@vuepress/shared-utils" "^1.2.0"
markdown-it-container "^2.0.0"
+vuepress-plugin-meilisearch@0.12.2:
+ version "0.12.2"
+ resolved "https://registry.yarnpkg.com/vuepress-plugin-meilisearch/-/vuepress-plugin-meilisearch-0.12.2.tgz#d130fa77a5f1ca5f8ae09af14e504f21779376d1"
+ integrity sha512-ug9+puCPySr/wJEnymWqFqbo3xsQfqXog2kFU8RPjrOQ20miQjgPYnF3u/8Pb+QT0s5zHA6ykT8lTBx0ovWsVw==
+ dependencies:
+ docs-searchbar.js "^2.1.1"
+
+vuepress-plugin-sitemap@2.3.1:
+ version "2.3.1"
+ resolved "https://registry.yarnpkg.com/vuepress-plugin-sitemap/-/vuepress-plugin-sitemap-2.3.1.tgz#51298aca77a5de96396fdbd1103e1637dd61ae6a"
+ integrity sha512-n+8lbukhrKrsI9H/EX0EBgkE1pn85LAQFvQ5dIvrZP4Kz6JxPOPPNTQmZMhahQV1tXbLZQCEN7A1WZH4x+arJQ==
+ dependencies:
+ sitemap "^3.0.0"
+
vuepress-plugin-smooth-scroll@^0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/vuepress-plugin-smooth-scroll/-/vuepress-plugin-smooth-scroll-0.0.3.tgz#6eff2d4c186cca917cc9f7df2b0af7de7c8c6438"
@@ -14758,6 +14816,11 @@ xml2js@0.4.23, xml2js@^0.4.19:
sax ">=0.6.0"
xmlbuilder "~11.0.0"
+xmlbuilder@^13.0.0:
+ version "13.0.2"
+ resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-13.0.2.tgz#02ae33614b6a047d1c32b5389c1fdacb2bce47a7"
+ integrity sha512-Eux0i2QdDYKbdbA6AM6xE4m6ZTZr4G4xF9kahI2ukSEMCzwce2eX9WlTI5J3s+NU7hpasFsr8hWIONae7LluAQ==
+
xmlbuilder@^15.1.1:
version "15.1.1"
resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5"