From 93167c346aa9df1b4e3cffff54591c5be2c52929 Mon Sep 17 00:00:00 2001 From: EsuRt <15177069+EsuRt@users.noreply.github.com> Date: Tue, 14 Jan 2020 11:43:36 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E8=B0=83=E6=95=B4=E4=B8=AD=E5=9B=BD?= =?UTF-8?q?=E6=94=BF=E5=BA=9C=E7=BD=91=E6=96=B0=E9=97=BB=E6=8E=92=E7=89=88?= =?UTF-8?q?=EF=BC=8C=E6=9B=B4=E6=94=B9=E8=8B=8F=E5=B7=9E=E6=94=BF=E5=BA=9C?= =?UTF-8?q?=E7=BD=91=E6=94=BF=E7=AD=96=E6=96=87=E4=BB=B6=E7=BD=91=E5=9D=80?= =?UTF-8?q?=20(#3710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/gov/news/index.js | 21 ++++++++++---------- lib/routes/gov/suzhou/doc.js | 37 +++++++++++++++++------------------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/lib/routes/gov/news/index.js b/lib/routes/gov/news/index.js index e29cacf76a..4425cf7a50 100644 --- a/lib/routes/gov/news/index.js +++ b/lib/routes/gov/news/index.js @@ -49,14 +49,14 @@ module.exports = async (ctx) => { item = $(item); let fullText = ''; let href = ''; - let contenlUrl = ''; + let contentUrl = ''; let description = ''; let fullTextData = ''; let fullTextGet = ''; if (item.html().search(/dysMiddleResultConItemTitle/g) !== -1) { - contenlUrl = item.find('a').attr('href'); - description = await ctx.cache.tryGet(contenlUrl, async () => { - fullTextGet = await got.get(contenlUrl); + contentUrl = item.find('a').attr('href'); + description = await ctx.cache.tryGet(contentUrl, async () => { + fullTextGet = await got.get(contentUrl); fullTextData = cheerio.load(fullTextGet.data); if (fullTextData.html().search(/pages_content/g) !== -1) { return fullTextData('.pages_content').html(); @@ -66,30 +66,31 @@ module.exports = async (ctx) => { }); } else { href = item.find('a').attr('href'); - contenlUrl = href.startsWith('/') ? `http://www.gov.cn${href}` : href; - description = await ctx.cache.tryGet(contenlUrl, async () => { - fullTextGet = await got.get(contenlUrl); + contentUrl = href.startsWith('/') ? `http://www.gov.cn${href}` : href; + description = await ctx.cache.tryGet(contentUrl, async () => { + fullTextGet = await got.get(contentUrl); fullTextData = cheerio.load(fullTextGet.data); const $1 = fullTextData.html(); - if (contenlUrl.search(/zhengceku/g) !== -1) { + if (contentUrl.search(/zhengceku/g) !== -1) { fullText = fullTextData('.pages_content').html(); } else { if ($1.search(/UCAP-CONTENT/g) === -1) { fullText = fullTextData('body').html(); } else { + fullTextData('.shuzi').remove(); fullText = fullTextData('#UCAP-CONTENT').html(); } } const regImg = /(img src=")(.*)(".*)/g; const regUrl = /(http.*\/)(content.*)/; - const urlSplit = contenlUrl.replace(regUrl, '$1'); + const urlSplit = contentUrl.replace(regUrl, '$1'); return fullText.replace(regImg, '$1' + urlSplit + '$2' + '$3'); }); } return { title: item.find('a').text(), description: description, - link: contenlUrl, + link: contentUrl, }; }) .get() diff --git a/lib/routes/gov/suzhou/doc.js b/lib/routes/gov/suzhou/doc.js index 7a82ceeace..f5763f96a1 100644 --- a/lib/routes/gov/suzhou/doc.js +++ b/lib/routes/gov/suzhou/doc.js @@ -2,41 +2,38 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); module.exports = async (ctx) => { - const link = 'http://www.zfxxgk.suzhou.gov.cn/sxqzf/szsrmzf/index_1073.html'; + const link = 'http://www.suzhou.gov.cn/szxxgk/front/xxgk_right.jsp?sitecode=szsrmzf'; const response = await got.get(link); const data = response.data; - const str = cheerio.load(data); - const strSelect = str('script:not([src])')[0].children[0].data; - const list = strSelect.match(/(url:".)(\/.*html)"/g); + const $ = cheerio.load(data); + const list = $('.tr_main_value_odd'); ctx.state.data = { title: '苏州市政府 - 政策公开文件', link: link, item: await Promise.all( - list && - list.slice(1, 10).map(async (index, item) => { + list + .slice(1, 12) + .map(async (index, item) => { + item = $(item); // 获取全文 - const contenlhref = list[item].replace(/(url:".)(\/.*html)"/, '$2'); - const contenlUrl = 'http://www.zfxxgk.suzhou.gov.cn/sxqzf/szsrmzf' + contenlhref; - const arr = await ctx.cache.tryGet(contenlUrl, async () => { - const fullText = await got.get(contenlUrl); + const contentUrl = 'http://www.suzhou.gov.cn' + item.find('a').attr('href'); + const arr = await ctx.cache.tryGet(contentUrl, async () => { + const fullText = await got.get(contentUrl); const fullTextData = cheerio.load(fullText.data); - fullTextData('h1').remove(); - const textSelect = fullTextData('body').html(); - - // 处理图片地址 - const regImg = /(src=")(\.\/)(.*\.jpg)/g; - const regpdf = /(href=")(\.\/)(.*\.pdf)/g; - const regUrl = /(http.*\/)(.*)(\.html)/; - const urlSplit = contenlUrl.replace(regUrl, '$1'); - return new Array(fullTextData('article h2').text(), textSelect.replace(regImg, '$1' + urlSplit + '$3').replace(regpdf, '$1' + urlSplit + '$3')); + const description = fullTextData('.content').html(); + const title = fullTextData('.title.bold') + .text() + .replace(/\s*/g, ''); + return new Array(title, description); }); return { title: arr[0], description: arr[1], - link: contenlUrl, + link: contentUrl, }; }) + .get() ), }; };