mirror of
https://github.com/DIYgod/RSSHub.git
synced 2026-09-24 23:12:34 +08:00
feat: work with hono and typescript
This commit is contained in:
DIYgod
@@ -1,4 +0,0 @@
|
||||
#!/usr/bin/env sh
|
||||
. "$(dirname -- "$0")/_/husky.sh"
|
||||
|
||||
npx lint-staged
|
||||
+349
-47
@@ -1,16 +1,321 @@
|
||||
require('dotenv').config();
|
||||
const randUserAgent = require('./utils/rand-user-agent');
|
||||
const got = require('got');
|
||||
import 'dotenv/config'
|
||||
import randUserAgent from '@/utils/rand-user-agent';
|
||||
import got from 'got';
|
||||
|
||||
let envs = process.env;
|
||||
let value;
|
||||
|
||||
type Config = {
|
||||
disallowRobot: boolean;
|
||||
enableCluster?: string;
|
||||
isPackage: boolean;
|
||||
nodeName?: string;
|
||||
puppeteerWSEndpoint?: string;
|
||||
chromiumExecutablePath?: string;
|
||||
connect: {
|
||||
port: number;
|
||||
socket: string | null;
|
||||
};
|
||||
listenInaddrAny: boolean;
|
||||
requestRetry: number;
|
||||
requestTimeout: number;
|
||||
ua: string;
|
||||
trueUA: string;
|
||||
allowOrigin?: string;
|
||||
cache: {
|
||||
type: string;
|
||||
requestTimeout: number;
|
||||
routeExpire: number;
|
||||
contentExpire: number;
|
||||
};
|
||||
memory: {
|
||||
max: number;
|
||||
};
|
||||
redis: {
|
||||
url: string;
|
||||
};
|
||||
proxyUri?: string;
|
||||
proxy: {
|
||||
protocol?: string;
|
||||
host?: string;
|
||||
port?: string;
|
||||
auth?: string;
|
||||
url_regex: string;
|
||||
};
|
||||
proxyStrategy: string;
|
||||
reverseProxyUrl?: string;
|
||||
pacUri?: string;
|
||||
pacScript?: string;
|
||||
authentication: {
|
||||
name: string;
|
||||
pass: string;
|
||||
};
|
||||
denylist?: string[];
|
||||
allowlist?: string[];
|
||||
allowLocalhost: boolean;
|
||||
accessKey?: string;
|
||||
debugInfo: string;
|
||||
loggerLevel: string;
|
||||
noLogfiles?: boolean;
|
||||
showLoggerTimestamp?: boolean;
|
||||
sentry: {
|
||||
dsn?: string;
|
||||
routeTimeout: number;
|
||||
};
|
||||
hotlink: {
|
||||
template?: string;
|
||||
includePaths?: string[];
|
||||
excludePaths?: string[];
|
||||
};
|
||||
feature: {
|
||||
allow_user_hotlink_template: boolean;
|
||||
filter_regex_engine: string;
|
||||
allow_user_supply_unsafe_domain: boolean;
|
||||
};
|
||||
suffix?: string;
|
||||
titleLengthLimit: number;
|
||||
openai: {
|
||||
apiKey?: string;
|
||||
model?: string;
|
||||
temperature?: number;
|
||||
maxTokens?: number;
|
||||
endpoint?: string;
|
||||
prompt?: string;
|
||||
};
|
||||
bilibili: {
|
||||
cookies: Record<string, string | undefined>;
|
||||
dmImgList?: string;
|
||||
};
|
||||
bitbucket: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
};
|
||||
btbyr: {
|
||||
host?: string;
|
||||
cookies?: string;
|
||||
};
|
||||
bupt: {
|
||||
portal_cookie?: string;
|
||||
};
|
||||
civitai: {
|
||||
cookie?: string;
|
||||
};
|
||||
dida365: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
};
|
||||
discord: {
|
||||
authorization?: string;
|
||||
};
|
||||
discourse: {
|
||||
config: Record<string, string | undefined>;
|
||||
};
|
||||
discuz: {
|
||||
cookies: Record<string, string | undefined>;
|
||||
};
|
||||
disqus: {
|
||||
api_key?: string;
|
||||
};
|
||||
douban: {
|
||||
cookie?: string;
|
||||
};
|
||||
ehentai: {
|
||||
ipb_member_id?: string;
|
||||
ipb_pass_hash?: string;
|
||||
sk?: string;
|
||||
igneous?: string;
|
||||
star?: string;
|
||||
img_proxy?: string;
|
||||
};
|
||||
email: {
|
||||
config: Record<string, string | undefined>;
|
||||
};
|
||||
fanbox: {
|
||||
session?: string;
|
||||
};
|
||||
fanfou: {
|
||||
consumer_key?: string;
|
||||
consumer_secret?: string;
|
||||
username?: string;
|
||||
password?: string;
|
||||
};
|
||||
fantia: {
|
||||
cookies?: string;
|
||||
};
|
||||
game4399: {
|
||||
cookie?: string;
|
||||
};
|
||||
github: {
|
||||
access_token?: string;
|
||||
};
|
||||
gitee: {
|
||||
access_token?: string;
|
||||
};
|
||||
google: {
|
||||
fontsApiKey?: string;
|
||||
};
|
||||
hefeng: {
|
||||
key?: string;
|
||||
};
|
||||
infzm: {
|
||||
cookie?: string;
|
||||
};
|
||||
initium: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
bearertoken?: string;
|
||||
iap_receipt?: string;
|
||||
};
|
||||
instagram: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
proxy?: string;
|
||||
cookie?: string;
|
||||
};
|
||||
iwara: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
};
|
||||
lastfm: {
|
||||
api_key?: string;
|
||||
};
|
||||
lightnovel: {
|
||||
cookie?: string;
|
||||
};
|
||||
manhuagui: {
|
||||
cookie?: string;
|
||||
};
|
||||
mastodon: {
|
||||
apiHost?: string;
|
||||
accessToken?: string;
|
||||
acctDomain?: string;
|
||||
};
|
||||
medium: {
|
||||
cookies: Record<string, string | undefined>;
|
||||
articleCookie?: string;
|
||||
};
|
||||
miniflux: {
|
||||
instance?: string;
|
||||
token?: string;
|
||||
};
|
||||
ncm: {
|
||||
cookies?: string;
|
||||
};
|
||||
newrank: {
|
||||
cookie?: string;
|
||||
};
|
||||
nga: {
|
||||
uid?: string;
|
||||
cid?: string;
|
||||
};
|
||||
nhentai: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
};
|
||||
notion: {
|
||||
key?: string;
|
||||
};
|
||||
pianyuan: {
|
||||
cookie?: string;
|
||||
};
|
||||
pixabay: {
|
||||
key?: string;
|
||||
};
|
||||
pixiv: {
|
||||
refreshToken?: string;
|
||||
bypassCdn?: boolean;
|
||||
bypassCdnHostname?: string;
|
||||
bypassCdnDoh?: string;
|
||||
imgProxy?: string;
|
||||
};
|
||||
pkubbs: {
|
||||
cookie?: string;
|
||||
};
|
||||
saraba1st: {
|
||||
cookie?: string;
|
||||
};
|
||||
sehuatang: {
|
||||
cookie?: string;
|
||||
};
|
||||
scboy: {
|
||||
token?: string;
|
||||
};
|
||||
scihub: {
|
||||
host?: string;
|
||||
};
|
||||
spotify: {
|
||||
clientId?: string;
|
||||
clientSecret?: string;
|
||||
refreshToken?: string;
|
||||
};
|
||||
telegram: {
|
||||
token?: string;
|
||||
};
|
||||
tophub: {
|
||||
cookie?: string;
|
||||
};
|
||||
twitter: {
|
||||
oauthTokens?: string[];
|
||||
oauthTokenSecrets?: string[];
|
||||
};
|
||||
weibo: {
|
||||
app_key?: string;
|
||||
app_secret?: string;
|
||||
cookies?: string;
|
||||
redirect_url?: string;
|
||||
};
|
||||
wenku8: {
|
||||
cookie?: string;
|
||||
};
|
||||
wordpress: {
|
||||
cdnUrl?: string;
|
||||
};
|
||||
xiaoyuzhou: {
|
||||
device_id?: string;
|
||||
refresh_token?: string;
|
||||
};
|
||||
ximalaya: {
|
||||
token?: string;
|
||||
};
|
||||
youtube: {
|
||||
key?: string;
|
||||
clientId?: string;
|
||||
clientSecret?: string;
|
||||
refreshToken?: string;
|
||||
};
|
||||
zhihu: {
|
||||
cookies?: string;
|
||||
};
|
||||
zodgame: {
|
||||
cookie?: string;
|
||||
};
|
||||
};
|
||||
|
||||
let _value: Config | undefined = undefined;
|
||||
|
||||
const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)';
|
||||
|
||||
const toBoolean = (value: string | undefined, defaultValue: boolean) => {
|
||||
if (value !== undefined) {
|
||||
return (value === '' || value === '0' || value === 'false') ? false : !!value;
|
||||
} else {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
const toInt = (value: string | undefined, defaultValue: number) => {
|
||||
if (value !== undefined) {
|
||||
return Number.parseInt(value);
|
||||
} else {
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
const calculateValue = () => {
|
||||
const bilibili_cookies = {};
|
||||
const email_config = {};
|
||||
const discuz_cookies = {};
|
||||
const medium_cookies = {};
|
||||
const discourse_config = {};
|
||||
const bilibili_cookies: Record<string, string | undefined> = {};
|
||||
const email_config: Record<string, string | undefined> = {};
|
||||
const discuz_cookies: Record<string, string | undefined> = {};
|
||||
const medium_cookies: Record<string, string | undefined> = {};
|
||||
const discourse_config: Record<string, string | undefined> = {};
|
||||
|
||||
for (const name in envs) {
|
||||
if (name.startsWith('BILIBILI_COOKIE_')) {
|
||||
@@ -27,39 +332,39 @@ const calculateValue = () => {
|
||||
medium_cookies[username] = envs[name];
|
||||
} else if (name.startsWith('DISCOURSE_CONFIG_')) {
|
||||
const id = name.slice('DISCOURSE_CONFIG_'.length);
|
||||
discourse_config[id] = JSON.parse(envs[name]);
|
||||
discourse_config[id] = JSON.parse(envs[name] || '{}');
|
||||
}
|
||||
}
|
||||
|
||||
value = {
|
||||
_value = {
|
||||
// app config
|
||||
disallowRobot: envs.DISALLOW_ROBOT !== '0' && envs.DISALLOW_ROBOT !== 'false',
|
||||
enableCluster: envs.ENABLE_CLUSTER,
|
||||
isPackage: envs.IS_PACKAGE,
|
||||
isPackage: !!envs.IS_PACKAGE,
|
||||
nodeName: envs.NODE_NAME,
|
||||
puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
|
||||
chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH,
|
||||
// network
|
||||
connect: {
|
||||
port: envs.PORT || 1200, // 监听端口
|
||||
port: toInt(envs.PORT, 1200), // 监听端口
|
||||
socket: envs.SOCKET || null, // 监听 Unix Socket, null 为禁用
|
||||
},
|
||||
listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
|
||||
requestRetry: Number.parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
|
||||
requestTimeout: Number.parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
|
||||
ua: envs.UA ?? (envs.NO_RANDOM_UA === 'true' || envs.NO_RANDOM_UA === '1' ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' })),
|
||||
listenInaddrAny: toBoolean(envs.LISTEN_INADDR_ANY, true), // 是否允许公网连接，取值 0 1
|
||||
requestRetry: toInt(envs.REQUEST_RETRY, 2), // 请求失败重试次数
|
||||
requestTimeout: toInt(envs.REQUEST_TIMEOUT, 30000), // Milliseconds to wait for the server to end the response before aborting the request
|
||||
ua: envs.UA ?? (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' })),
|
||||
trueUA: TRUE_UA,
|
||||
// cors request
|
||||
allowOrigin: envs.ALLOW_ORIGIN,
|
||||
// cache
|
||||
cache: {
|
||||
type: envs.CACHE_TYPE === undefined ? 'memory' : envs.CACHE_TYPE, // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存
|
||||
requestTimeout: Number.parseInt(envs.CACHE_REQUEST_TIMEOUT) || 60,
|
||||
routeExpire: Number.parseInt(envs.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间，单位为秒
|
||||
contentExpire: Number.parseInt(envs.CACHE_CONTENT_EXPIRE) || 1 * 60 * 60, // 不变内容缓存时间，单位为秒
|
||||
type: envs.CACHE_TYPE || 'memory', // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存
|
||||
requestTimeout: toInt(envs.CACHE_REQUEST_TIMEOUT, 60),
|
||||
routeExpire: toInt(envs.CACHE_EXPIRE, 5 * 60), // 路由缓存时间，单位为秒
|
||||
contentExpire: toInt(envs.CACHE_CONTENT_EXPIRE, 1 * 60 * 60), // 不变内容缓存时间，单位为秒
|
||||
},
|
||||
memory: {
|
||||
max: Number.parseInt(envs.MEMORY_MAX) || Math.pow(2, 8), // The maximum number of items that remain in the cache. This must be a positive finite intger.
|
||||
max: toInt(envs.MEMORY_MAX, Math.pow(2, 8)), // The maximum number of items that remain in the cache. This must be a positive finite intger.
|
||||
// https://github.com/isaacs/node-lru-cache#options
|
||||
},
|
||||
redis: {
|
||||
@@ -84,38 +389,38 @@ const calculateValue = () => {
|
||||
pass: envs.HTTP_BASIC_AUTH_PASS || 'passw0rd',
|
||||
},
|
||||
// access control
|
||||
denylist: envs.DENYLIST && envs.DENYLIST.split(','),
|
||||
allowlist: envs.ALLOWLIST && envs.ALLOWLIST.split(','),
|
||||
allowLocalhost: envs.ALLOW_LOCALHOST,
|
||||
denylist: envs.DENYLIST ? envs.DENYLIST.split(',') : undefined,
|
||||
allowlist: envs.ALLOWLIST ? envs.ALLOWLIST.split(',') : undefined,
|
||||
allowLocalhost: toBoolean(envs.ALLOW_LOCALHOST, false),
|
||||
accessKey: envs.ACCESS_KEY,
|
||||
// logging
|
||||
// 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示
|
||||
debugInfo: envs.DEBUG_INFO || 'true',
|
||||
loggerLevel: envs.LOGGER_LEVEL || 'info',
|
||||
noLogfiles: envs.NO_LOGFILES,
|
||||
showLoggerTimestamp: envs.SHOW_LOGGER_TIMESTAMP,
|
||||
noLogfiles: toBoolean(envs.NO_LOGFILES, false),
|
||||
showLoggerTimestamp: toBoolean(envs.SHOW_LOGGER_TIMESTAMP, false),
|
||||
sentry: {
|
||||
dsn: envs.SENTRY,
|
||||
routeTimeout: Number.parseInt(envs.SENTRY_ROUTE_TIMEOUT) || 30000,
|
||||
routeTimeout: toInt(envs.SENTRY_ROUTE_TIMEOUT, 30000),
|
||||
},
|
||||
// feed config
|
||||
hotlink: {
|
||||
template: envs.HOTLINK_TEMPLATE,
|
||||
includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','),
|
||||
excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','),
|
||||
includePaths: envs.HOTLINK_INCLUDE_PATHS ? envs.HOTLINK_INCLUDE_PATHS.split(',') : undefined,
|
||||
excludePaths: envs.HOTLINK_EXCLUDE_PATHS ? envs.HOTLINK_EXCLUDE_PATHS.split(',') : undefined,
|
||||
},
|
||||
feature: {
|
||||
allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
|
||||
allow_user_hotlink_template: toBoolean(envs.ALLOW_USER_HOTLINK_TEMPLATE, false),
|
||||
filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
|
||||
allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true',
|
||||
allow_user_supply_unsafe_domain: toBoolean(envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN, false),
|
||||
},
|
||||
suffix: envs.SUFFIX,
|
||||
titleLengthLimit: Number.parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
|
||||
titleLengthLimit: toInt(envs.TITLE_LENGTH_LIMIT, 150),
|
||||
openai: {
|
||||
apiKey: envs.OPENAI_API_KEY,
|
||||
model: envs.OPENAI_MODEL || 'gpt-3.5-turbo-16k',
|
||||
temperature: envs.OPENAI_TEMPERATURE || 0.2,
|
||||
maxTokens: envs.OPENAI_MAX_TOKENS || null,
|
||||
temperature: toInt(envs.OPENAI_TEMPERATURE, 0.2),
|
||||
maxTokens: toInt(envs.OPENAI_MAX_TOKENS, 0) || undefined,
|
||||
endpoint: envs.OPENAI_API_ENDPOINT || 'https://api.openai.com/v1',
|
||||
prompt: envs.OPENAI_PROMPT || 'Please summarize the following article and reply with markdown format.',
|
||||
},
|
||||
@@ -263,7 +568,7 @@ const calculateValue = () => {
|
||||
},
|
||||
pixiv: {
|
||||
refreshToken: envs.PIXIV_REFRESHTOKEN,
|
||||
bypassCdn: envs.PIXIV_BYPASS_CDN && envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
|
||||
bypassCdn: toBoolean(envs.PIXIV_BYPASS_CDN, false),
|
||||
bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
|
||||
bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
|
||||
imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re',
|
||||
@@ -340,20 +645,17 @@ if (envs.REMOTE_CONFIG) {
|
||||
if (data) {
|
||||
envs = Object.assign(envs, data);
|
||||
calculateValue();
|
||||
require('@/utils/logger').info('Remote config loaded.');
|
||||
require('@/utils/logger').default.info('Remote config loaded.');
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
require('@/utils/logger').error('Remote config load failed.', error);
|
||||
require('@/utils/logger').default.error('Remote config load failed.', error);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
set: (env) => {
|
||||
envs = Object.assign(process.env, env);
|
||||
calculateValue();
|
||||
},
|
||||
get value() {
|
||||
return value;
|
||||
},
|
||||
};
|
||||
export const config: Config = _value!;
|
||||
|
||||
export const setConfig = (env: Partial<Config>) => {
|
||||
envs = Object.assign(process.env, env);
|
||||
calculateValue();
|
||||
}
|
||||
@@ -1,3 +1,3 @@
|
||||
class RequestInProgressError extends Error {}
|
||||
|
||||
module.exports = RequestInProgressError;
|
||||
export default RequestInProgressError;
|
||||
@@ -1,3 +0,0 @@
|
||||
module.exports = {
|
||||
RequestInProgressError: require('./RequestInProgress'),
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
import _RequestInProgressError from './RequestInProgress';
|
||||
|
||||
export const RequestInProgressError = _RequestInProgressError;
|
||||
@@ -0,0 +1,51 @@
|
||||
import { serve } from '@hono/node-server'
|
||||
import { Handler, Hono } from 'hono'
|
||||
|
||||
import cache from '@/middleware/cache'
|
||||
import template from '@/middleware/template'
|
||||
import onerror from '@/middleware/onerror'
|
||||
import accessControl from '@/middleware/access-control'
|
||||
import debug from '@/middleware/debug'
|
||||
import header from '@/middleware/header'
|
||||
import antiHotlink from '@/middleware/anti-hotlink'
|
||||
import parameter from '@/middleware/parameter'
|
||||
|
||||
import routes from '@/routes'
|
||||
import { config } from '@/config'
|
||||
|
||||
const app = new Hono()
|
||||
|
||||
app.use('*', onerror);
|
||||
app.use('*', accessControl);
|
||||
app.use('*', debug);
|
||||
app.use('*', header);
|
||||
app.use('*', template);
|
||||
app.use('*', antiHotlink);
|
||||
app.use('*', parameter);
|
||||
app.use('*', cache);
|
||||
|
||||
for (const name in routes) {
|
||||
const subApp = app.basePath(`/${name}`)
|
||||
routes[name]({
|
||||
get: (path, handler) => {
|
||||
const wrapedHandler: Handler = async (ctx, ...args) => {
|
||||
if (!ctx.get('data')) {
|
||||
await handler(ctx, ...args)
|
||||
}
|
||||
}
|
||||
subApp.get(path, wrapedHandler)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
app.get('/', (c) => {
|
||||
return c.text('Hello Hono!')
|
||||
})
|
||||
|
||||
const port = config.connect.port
|
||||
console.log(`Server is running on port ${port}`)
|
||||
|
||||
serve({
|
||||
fetch: app.fetch,
|
||||
port
|
||||
})
|
||||
@@ -1,9 +1,11 @@
|
||||
const config = require('@/config').value;
|
||||
const md5 = require('@/utils/md5');
|
||||
const isLocalhost = require('is-localhost-ip');
|
||||
import type { MiddlewareHandler, Context } from 'hono';
|
||||
import { config } from '@/config';
|
||||
import md5 from '@/utils/md5';
|
||||
import isLocalhost from 'is-localhost-ip';
|
||||
import { getIp } from '@/utils/helpers';
|
||||
|
||||
const reject = (ctx) => {
|
||||
ctx.response.status = 403;
|
||||
const reject = (ctx: Context) => {
|
||||
ctx.status(403);
|
||||
|
||||
throw new Error('Authentication failed. Access denied.');
|
||||
};
|
||||
@@ -11,7 +13,7 @@ const reject = (ctx) => {
|
||||
const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/;
|
||||
const cidrPattern = /((?:\d{1,3}\.){3}\d{1,3})\/(\d{1,2})/;
|
||||
|
||||
const ipInCidr = (cidr, ip) => {
|
||||
const ipInCidr = (cidr: string, ip: string) => {
|
||||
const cidrMatch = cidr.match(cidrPattern);
|
||||
const ipMatch = ip.match(ipv4Pattern);
|
||||
if (!cidrMatch || !ipMatch) {
|
||||
@@ -23,25 +25,25 @@ const ipInCidr = (cidr, ip) => {
|
||||
return cidrIpBits === ipBits;
|
||||
};
|
||||
|
||||
const ipv4ToBitsring = (ip) =>
|
||||
const ipv4ToBitsring = (ip: string) =>
|
||||
ip
|
||||
.split('.')
|
||||
.map((part) => ('00000000' + Number.parseInt(part).toString(2)).slice(-8))
|
||||
.join('');
|
||||
|
||||
module.exports = async (ctx, next) => {
|
||||
const ip = ctx.ips[0] || ctx.ip;
|
||||
const requestPath = ctx.request.path;
|
||||
const requestUA = ctx.request.header['user-agent'];
|
||||
const accessKey = ctx.query.key;
|
||||
const accessCode = ctx.query.code;
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
const ip = getIp(ctx);
|
||||
const requestPath = ctx.req.path;
|
||||
const requestUA = ctx.req.header('user-agent');
|
||||
const accessKey = ctx.req.query('key');
|
||||
const accessCode = ctx.req.query('code');
|
||||
|
||||
const isControlled = config.accessKey || config.allowlist || config.denylist;
|
||||
|
||||
const allowLocalhost = config.allowLocalhost && (await isLocalhost(ip));
|
||||
const allowLocalhost = config.allowLocalhost && ip && (await isLocalhost(ip));
|
||||
|
||||
const grant = async () => {
|
||||
if (ctx.response.status !== 403) {
|
||||
if (ctx.res.status !== 403) {
|
||||
await next();
|
||||
}
|
||||
};
|
||||
@@ -57,14 +59,16 @@ module.exports = async (ctx, next) => {
|
||||
return grant();
|
||||
}
|
||||
|
||||
if (config.allowlist && config.allowlist.some((item) => ip.includes(item) || ipInCidr(item, ip) || requestPath.includes(item) || requestUA.includes(item))) {
|
||||
if (config.allowlist && config.allowlist.some((item) => ip?.includes(item) || (ip && ipInCidr(item, ip)) || requestPath.includes(item) || requestUA?.includes(item))) {
|
||||
return grant();
|
||||
}
|
||||
|
||||
if (config.denylist && !config.denylist.some((item) => ip.includes(item) || ipInCidr(item, ip) || requestPath.includes(item) || requestUA.includes(item))) {
|
||||
if (config.denylist && !config.denylist.some((item) => ip?.includes(item) || (ip && ipInCidr(item, ip)) || requestPath.includes(item) || requestUA?.includes(item))) {
|
||||
return grant();
|
||||
}
|
||||
|
||||
reject(ctx);
|
||||
}
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
@@ -1,15 +1,16 @@
|
||||
const config = require('@/config').value;
|
||||
const cheerio = require('cheerio');
|
||||
const logger = require('@/utils/logger');
|
||||
const path = require('path');
|
||||
const { art } = require('@/utils/render');
|
||||
import { config } from '@/config';
|
||||
import { load, type CheerioAPI } from 'cheerio';
|
||||
import logger from '@/utils/logger';
|
||||
import * as path from 'node:path';
|
||||
import render from '@/utils/render';
|
||||
import { type MiddlewareHandler } from 'hono';
|
||||
|
||||
const templateRegex = /\${([^{}]+)}/g;
|
||||
const allowedUrlProperties = new Set(['hash', 'host', 'hostname', 'href', 'origin', 'password', 'pathname', 'port', 'protocol', 'search', 'searchParams', 'username']);
|
||||
const IframeWrapperTemplate = path.join(__dirname, 'templates/iframe.art');
|
||||
|
||||
// match path or sub-path
|
||||
const matchPath = (path, paths) => {
|
||||
const matchPath = (path: string, paths: string[]) => {
|
||||
for (const p of paths) {
|
||||
if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) {
|
||||
return true;
|
||||
@@ -19,13 +20,13 @@ const matchPath = (path, paths) => {
|
||||
};
|
||||
|
||||
// return ture if the path needs to be processed
|
||||
const filterPath = (path) => {
|
||||
const filterPath = (path: string) => {
|
||||
const include = config.hotlink.includePaths;
|
||||
const exclude = config.hotlink.excludePaths;
|
||||
return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude));
|
||||
};
|
||||
|
||||
const interpolate = (str, obj) =>
|
||||
const interpolate = (str: string, obj: Record<string, any>) =>
|
||||
str.replaceAll(templateRegex, (_, prop) => {
|
||||
let needEncode = false;
|
||||
if (prop.endsWith('_ue')) {
|
||||
@@ -35,7 +36,7 @@ const interpolate = (str, obj) =>
|
||||
}
|
||||
return needEncode ? encodeURIComponent(obj[prop]) : obj[prop];
|
||||
});
|
||||
const parseUrl = (str) => {
|
||||
const parseUrl = (str: string) => {
|
||||
let url;
|
||||
try {
|
||||
url = new URL(str);
|
||||
@@ -45,7 +46,7 @@ const parseUrl = (str) => {
|
||||
|
||||
return url;
|
||||
};
|
||||
const replaceUrls = ($, selector, template, attribute = 'src') => {
|
||||
const replaceUrls = ($: CheerioAPI, selector: string, template: string, attribute = 'src') => {
|
||||
$(selector).each(function () {
|
||||
const old_src = $(this).attr(attribute);
|
||||
if (old_src) {
|
||||
@@ -58,15 +59,15 @@ const replaceUrls = ($, selector, template, attribute = 'src') => {
|
||||
});
|
||||
};
|
||||
|
||||
const wrapWithIframe = ($, selector) => {
|
||||
const wrapWithIframe = ($: CheerioAPI, selector: string) => {
|
||||
$(selector).each((_, elem) => {
|
||||
elem = $(elem);
|
||||
elem.replaceWith(art(IframeWrapperTemplate, { content: elem.toString() }));
|
||||
const $elem = $(elem);
|
||||
$elem.replaceWith(render.art(IframeWrapperTemplate, { content: elem.toString() }));
|
||||
});
|
||||
};
|
||||
|
||||
const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap_multimedia_in_iframe) => {
|
||||
const $ = cheerio.load(html, undefined, false);
|
||||
const process = (html: string, image_hotlink_template?: string, multimedia_hotlink_template?: string, wrap_multimedia_in_iframe?: boolean) => {
|
||||
const $ = load(html, undefined, false);
|
||||
if (image_hotlink_template) {
|
||||
replaceUrls($, 'img, picture > source', image_hotlink_template);
|
||||
replaceUrls($, 'video[poster]', image_hotlink_template, 'poster');
|
||||
@@ -83,7 +84,7 @@ const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap
|
||||
return $.html();
|
||||
};
|
||||
|
||||
const validateTemplate = (template) => {
|
||||
const validateTemplate = (template?: string) => {
|
||||
if (!template) {
|
||||
return;
|
||||
}
|
||||
@@ -95,12 +96,12 @@ const validateTemplate = (template) => {
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = async (ctx, next) => {
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
await next();
|
||||
|
||||
let image_hotlink_template;
|
||||
let multimedia_hotlink_template;
|
||||
const shouldWrapInIframe = ctx.query.wrap_multimedia_in_iframe === '1';
|
||||
const shouldWrapInIframe = ctx.req.query('wrap_multimedia_in_iframe') === '1';
|
||||
|
||||
// Read params if enabled
|
||||
if (config.feature.allow_user_hotlink_template) {
|
||||
@@ -109,13 +110,13 @@ module.exports = async (ctx, next) => {
|
||||
// A risk is that the media URLs will be replaced by user-supplied templates,
|
||||
// so a user could literally take the control of "where are the media from",
|
||||
// but only in their personal-use feed URL.
|
||||
multimedia_hotlink_template = ctx.query.multimedia_hotlink_template;
|
||||
image_hotlink_template = ctx.query.image_hotlink_template;
|
||||
multimedia_hotlink_template = ctx.req.query('multimedia_hotlink_template');
|
||||
image_hotlink_template = ctx.req.query('image_hotlink_template');
|
||||
}
|
||||
|
||||
// Force config hotlink template on conflict
|
||||
if (config.hotlink.template) {
|
||||
image_hotlink_template = filterPath(ctx.request.path) ? config.hotlink.template : undefined;
|
||||
image_hotlink_template = filterPath(ctx.req.path) ? config.hotlink.template : undefined;
|
||||
}
|
||||
|
||||
if (!image_hotlink_template && !multimedia_hotlink_template && !shouldWrapInIframe) {
|
||||
@@ -129,17 +130,22 @@ module.exports = async (ctx, next) => {
|
||||
// and here we will only check them in description.
|
||||
// Use Cheerio to load the description as html and filter all
|
||||
// image link
|
||||
if (ctx.state.data) {
|
||||
if (ctx.state.data.description) {
|
||||
ctx.state.data.description = process(ctx.state.data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
|
||||
const data = ctx.get('data');
|
||||
if (data) {
|
||||
if (data.description) {
|
||||
data.description = process(data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
|
||||
}
|
||||
|
||||
if (ctx.state.data.item) {
|
||||
for (const item of ctx.state.data.item) {
|
||||
if (data.item) {
|
||||
for (const item of data.item) {
|
||||
if (item.description) {
|
||||
item.description = process(item.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ctx.set('data', data);
|
||||
}
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
@@ -0,0 +1,68 @@
|
||||
import xxhash from 'xxhash-wasm';
|
||||
import type { MiddlewareHandler } from 'hono';
|
||||
|
||||
import { config } from '@/config';
|
||||
import { RequestInProgressError } from '@/errors';
|
||||
import cacheModule from '@/utils/cache/index'
|
||||
|
||||
// only give cache string, as the `!` condition tricky
|
||||
// md5 is used to shrink key size
|
||||
// plz, write these tips in comments!
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
const { h64ToString } = await xxhash();
|
||||
const key = 'rsshub:koa-redis-cache:' + h64ToString(ctx.req.path);
|
||||
const controlKey = 'rsshub:path-requested:' + h64ToString(ctx.req.path);
|
||||
|
||||
if (!cacheModule.status.available) {
|
||||
await next();
|
||||
return;
|
||||
}
|
||||
|
||||
const isRequesting = await cacheModule.globalCache.get(controlKey);
|
||||
|
||||
if (isRequesting === '1') {
|
||||
throw new RequestInProgressError('This path is currently fetching, please come back later!');
|
||||
}
|
||||
|
||||
try {
|
||||
const value = await cacheModule.globalCache.get(key);
|
||||
|
||||
if (value) {
|
||||
ctx.status(200)
|
||||
if (config.cache.type === 'redis') {
|
||||
ctx.header('X-Koa-Redis-Cache', 'true')
|
||||
} else if (config.cache.type === 'memory') {
|
||||
ctx.header('X-Koa-Memory-Cache', 'true')
|
||||
}
|
||||
ctx.set('data', JSON.parse(value))
|
||||
await next();
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Doesn't hit the cache? We need to let others know!
|
||||
await cacheModule.globalCache.set(controlKey, '1', config.cache.requestTimeout);
|
||||
|
||||
try {
|
||||
await next();
|
||||
} catch (error) {
|
||||
await cacheModule.globalCache.set(controlKey, '0', config.cache.requestTimeout);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const data = ctx.get('data')
|
||||
if (ctx.res.headers.get('Cache-Control') !== 'no-cache' && data) {
|
||||
data.lastBuildDate = new Date().toUTCString();
|
||||
ctx.set('data', data)
|
||||
const body = JSON.stringify(data);
|
||||
await cacheModule.globalCache.set(key, body, config.cache.routeExpire);
|
||||
}
|
||||
|
||||
// We need to let it go, even no cache set.
|
||||
// Wait to set cache so the next request could be handled correctly
|
||||
await cacheModule.globalCache.set(controlKey, '0', config.cache.requestTimeout);
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
Vendored
-139
@@ -1,139 +0,0 @@
|
||||
const xxhash = require('xxhash-wasm');
|
||||
const config = require('@/config').value;
|
||||
const logger = require('@/utils/logger');
|
||||
const { RequestInProgressError } = require('@/errors');
|
||||
|
||||
const globalCache = {
|
||||
get: () => null,
|
||||
set: () => null,
|
||||
};
|
||||
|
||||
let cacheModule = {
|
||||
get: () => null,
|
||||
set: () => null,
|
||||
status: { available: false },
|
||||
clients: {},
|
||||
};
|
||||
|
||||
if (config.cache.type === 'redis') {
|
||||
cacheModule = require('./redis');
|
||||
const { redisClient } = cacheModule.clients;
|
||||
globalCache.get = async (key) => {
|
||||
if (key && cacheModule.status.available) {
|
||||
const value = await redisClient.get(key);
|
||||
return value;
|
||||
}
|
||||
};
|
||||
globalCache.set = cacheModule.set;
|
||||
} else if (config.cache.type === 'memory') {
|
||||
cacheModule = require('./memory');
|
||||
const { memoryCache } = cacheModule.clients;
|
||||
globalCache.get = (key) => {
|
||||
if (key && cacheModule.status.available) {
|
||||
return memoryCache.get(key, { updateAgeOnGet: false });
|
||||
}
|
||||
};
|
||||
globalCache.set = (key, value, maxAge) => {
|
||||
if (!value || value === 'undefined') {
|
||||
value = '';
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
value = JSON.stringify(value);
|
||||
}
|
||||
if (key) {
|
||||
return memoryCache.set(key, value, { ttl: maxAge * 1000 });
|
||||
}
|
||||
};
|
||||
} else {
|
||||
logger.error('Cache not available, concurrent requests are not limited. This could lead to bad behavior.');
|
||||
}
|
||||
|
||||
// only give cache string, as the `!` condition tricky
|
||||
// md5 is used to shrink key size
|
||||
// plz, write these tips in comments!
|
||||
module.exports = function (app) {
|
||||
const { get, set, status } = cacheModule;
|
||||
app.context.cache = {
|
||||
...cacheModule,
|
||||
tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => {
|
||||
if (typeof key !== 'string') {
|
||||
throw new TypeError('Cache key must be a string');
|
||||
}
|
||||
let v = await get(key, refresh);
|
||||
if (v) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(v);
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
if (parsed) {
|
||||
v = parsed;
|
||||
}
|
||||
} else {
|
||||
v = await getValueFunc();
|
||||
set(key, v, maxAge);
|
||||
}
|
||||
|
||||
return v;
|
||||
},
|
||||
globalCache,
|
||||
};
|
||||
|
||||
return async (ctx, next) => {
|
||||
const { h64ToString } = await xxhash();
|
||||
const key = 'rsshub:koa-redis-cache:' + h64ToString(ctx.request.path);
|
||||
const controlKey = 'rsshub:path-requested:' + h64ToString(ctx.request.path);
|
||||
|
||||
if (!status.available) {
|
||||
return next();
|
||||
}
|
||||
|
||||
const isRequesting = await globalCache.get(controlKey);
|
||||
|
||||
if (isRequesting === '1') {
|
||||
throw new RequestInProgressError('This path is currently fetching, please come back later!');
|
||||
}
|
||||
|
||||
try {
|
||||
const value = await globalCache.get(key);
|
||||
|
||||
if (value) {
|
||||
ctx.response.status = 200;
|
||||
if (config.cache.type === 'redis') {
|
||||
ctx.response.set({
|
||||
'X-Koa-Redis-Cache': 'true',
|
||||
});
|
||||
} else if (config.cache.type === 'memory') {
|
||||
ctx.response.set({
|
||||
'X-Koa-Memory-Cache': 'true',
|
||||
});
|
||||
}
|
||||
ctx.state.data = JSON.parse(value);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
//
|
||||
}
|
||||
|
||||
// Doesn't hit the cache? We need to let others know!
|
||||
await globalCache.set(controlKey, '1', config.cache.requestTimeout);
|
||||
|
||||
try {
|
||||
await next();
|
||||
} catch (error) {
|
||||
await globalCache.set(controlKey, '0', config.cache.requestTimeout);
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) {
|
||||
ctx.state.data.lastBuildDate = new Date().toUTCString();
|
||||
const body = JSON.stringify(ctx.state.data);
|
||||
await globalCache.set(key, body, config.cache.routeExpire);
|
||||
}
|
||||
|
||||
// We need to let it go, even no cache set.
|
||||
// Wait to set cache so the next request could be handled correctly
|
||||
await globalCache.set(controlKey, '0', config.cache.requestTimeout);
|
||||
};
|
||||
};
|
||||
@@ -1,25 +0,0 @@
|
||||
module.exports = async (ctx, next) => {
|
||||
if (!ctx.debug.paths[ctx.request.path]) {
|
||||
ctx.debug.paths[ctx.request.path] = 0;
|
||||
}
|
||||
ctx.debug.paths[ctx.request.path]++;
|
||||
|
||||
ctx.debug.request++;
|
||||
|
||||
await next();
|
||||
|
||||
if (!ctx.debug.routes[ctx._matchedRoute]) {
|
||||
ctx._matchedRoute && (ctx.debug.routes[ctx._matchedRoute] = 0);
|
||||
}
|
||||
ctx._matchedRoute && ctx.debug.routes[ctx._matchedRoute]++;
|
||||
|
||||
if (ctx.response.get('X-Koa-Redis-Cache') || ctx.response.get('X-Koa-Memory-Cache')) {
|
||||
ctx.debug.hitCache++;
|
||||
}
|
||||
|
||||
ctx.state.debuged = true;
|
||||
|
||||
if (ctx.status === 304) {
|
||||
ctx.debug.etag++;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,43 @@
|
||||
import { MiddlewareHandler } from "hono";
|
||||
import { getRouteNameFromPath } from '@/utils/helpers';
|
||||
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
const debug = Object.assign({
|
||||
hitCache: 0,
|
||||
request: 0,
|
||||
etag: 0,
|
||||
paths: [],
|
||||
routes: [],
|
||||
errorPaths: [],
|
||||
errorRoutes: [],
|
||||
}, ctx.get('debug'));
|
||||
|
||||
if (!debug.paths[ctx.req.path]) {
|
||||
debug.paths[ctx.req.path] = 0;
|
||||
}
|
||||
debug.paths[ctx.req.path]++;
|
||||
|
||||
debug.request++;
|
||||
|
||||
await next();
|
||||
|
||||
const routeName = getRouteNameFromPath(ctx.req.path);
|
||||
if (routeName) {
|
||||
if (!debug.routes[routeName]) {
|
||||
debug.routes[routeName] = 0;
|
||||
}
|
||||
debug.routes[routeName]++;
|
||||
}
|
||||
|
||||
if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) {
|
||||
debug.hitCache++;
|
||||
}
|
||||
|
||||
ctx.set('debuged', true);
|
||||
|
||||
if (ctx.res.status === 304) {
|
||||
debug.etag++;
|
||||
}
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
@@ -1,45 +0,0 @@
|
||||
const etagCalculate = require('etag');
|
||||
const logger = require('@/utils/logger');
|
||||
const config = require('@/config').value;
|
||||
const headers = {
|
||||
'Access-Control-Allow-Methods': 'GET',
|
||||
'Content-Type': 'application/xml; charset=utf-8',
|
||||
'Cache-Control': `public, max-age=${config.cache.routeExpire}`,
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
};
|
||||
if (config.nodeName) {
|
||||
headers['RSSHub-Node'] = config.nodeName;
|
||||
}
|
||||
|
||||
module.exports = async (ctx, next) => {
|
||||
logger.info(`${ctx.url}, user IP: ${ctx.ips[0] || ctx.ip}`);
|
||||
ctx.set(headers);
|
||||
ctx.set({
|
||||
'Access-Control-Allow-Origin': config.allowOrigin || ctx.host,
|
||||
});
|
||||
|
||||
await next();
|
||||
|
||||
if (!ctx.body || typeof ctx.body !== 'string' || ctx.response.get('ETag')) {
|
||||
return;
|
||||
}
|
||||
|
||||
const status = Math.trunc(ctx.status / 100);
|
||||
if (2 !== status) {
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.set('ETag', etagCalculate(ctx.body.replace(/<lastBuildDate>(.*)<\/lastBuildDate>/, '').replace(/<atom:link(.*)\/>/, '')));
|
||||
|
||||
if (ctx.fresh) {
|
||||
ctx.status = 304;
|
||||
ctx.body = null;
|
||||
} else {
|
||||
const match = ctx.body.match(/<lastBuildDate>(.*)<\/lastBuildDate>/);
|
||||
if (match) {
|
||||
ctx.set({
|
||||
'Last-Modified': match[1],
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,59 @@
|
||||
import { getIp } from "@/utils/helpers";
|
||||
import { MiddlewareHandler } from "hono";
|
||||
import etagCalculate from "etag";
|
||||
import logger from "@/utils/logger";
|
||||
import { config } from "@/config";
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Access-Control-Allow-Methods': 'GET',
|
||||
'Content-Type': 'application/xml; charset=utf-8',
|
||||
'Cache-Control': `public, max-age=${config.cache.routeExpire}`,
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
};
|
||||
if (config.nodeName) {
|
||||
headers['RSSHub-Node'] = config.nodeName;
|
||||
}
|
||||
|
||||
function etagMatches(etag: string, ifNoneMatch: string | null) {
|
||||
return ifNoneMatch != null && ifNoneMatch.split(/,\s*/).indexOf(etag) > -1
|
||||
}
|
||||
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
const ip = getIp(ctx);
|
||||
logger.info(`${ctx.req.url}, user IP: ${ip}`);
|
||||
|
||||
for (const key in headers) {
|
||||
ctx.header(key, headers[key])
|
||||
}
|
||||
ctx.header('Access-Control-Allow-Origin', config.allowOrigin || new URL(ctx.req.url).host);
|
||||
|
||||
await next();
|
||||
|
||||
if (!ctx.res.body || ctx.res.headers.get('ETag')) {
|
||||
return;
|
||||
}
|
||||
|
||||
const status = Math.trunc(ctx.res.status / 100);
|
||||
if (2 !== status) {
|
||||
return;
|
||||
}
|
||||
|
||||
const res = ctx.res as Response;
|
||||
const body = await res.clone().text();
|
||||
const etag = etagCalculate(body.replace(/<lastBuildDate>(.*)<\/lastBuildDate>/, '').replace(/<atom:link(.*)\/>/, ''));
|
||||
|
||||
ctx.set('ETag', etag);
|
||||
|
||||
const ifNoneMatch = ctx.req.header('If-None-Match') ?? null;
|
||||
if (etagMatches(etag, ifNoneMatch)) {
|
||||
ctx.status(304);
|
||||
ctx.body(null);
|
||||
} else {
|
||||
const match = body.match(/<lastBuildDate>(.*)<\/lastBuildDate>/);
|
||||
if (match) {
|
||||
ctx.header('Last-Modified', match[1]);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
@@ -1,119 +0,0 @@
|
||||
const logger = require('@/utils/logger');
|
||||
const config = require('@/config').value;
|
||||
const art = require('art-template');
|
||||
const path = require('path');
|
||||
|
||||
const { RequestInProgressError } = require('@/errors');
|
||||
|
||||
let Sentry;
|
||||
let gitHash;
|
||||
|
||||
if (config.sentry.dsn) {
|
||||
Sentry = Sentry || require('@sentry/node');
|
||||
Sentry.init({
|
||||
dsn: config.sentry.dsn,
|
||||
});
|
||||
Sentry.getCurrentScope().setTag('node_name', config.nodeName);
|
||||
|
||||
logger.info('Sentry inited.');
|
||||
}
|
||||
|
||||
try {
|
||||
gitHash = require('git-rev-sync').short();
|
||||
} catch {
|
||||
gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GIT_COMMIT_SHA && process.env.VERCEL_GIT_COMMIT_SHA.slice(0, 7)) || 'unknown';
|
||||
}
|
||||
|
||||
module.exports = async (ctx, next) => {
|
||||
try {
|
||||
const time = Date.now();
|
||||
await next();
|
||||
if (config.sentry.dsn && Date.now() - time >= config.sentry.routeTimeout) {
|
||||
Sentry.withScope((scope) => {
|
||||
scope.setTag('route', ctx._matchedRoute);
|
||||
scope.setTag('name', ctx.request.path.split('/')[1]);
|
||||
scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, ctx.request));
|
||||
Sentry.captureException(new Error('Route Timeout'));
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error && !error.stack.split('\n')[1].includes('lib/middleware/parameter.js')) {
|
||||
// Append v2 route path if a route throws an error
|
||||
// since koa-mount will remove the mount path from ctx.request.path
|
||||
// https://github.com/koajs/mount/issues/62
|
||||
ctx.request.path = (ctx.mountPath ?? '') + ctx.request.path;
|
||||
ctx._matchedRoute = ctx._matchedRoute ? (ctx.mountPath ?? '') + ctx._matchedRoute : ctx._matchedRoute;
|
||||
}
|
||||
|
||||
let message = error;
|
||||
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
|
||||
message = `${error.message}: target website might be blocking our access, you can <a href="https://docs.rsshub.app/install/">host your own RSSHub instance</a> for a better usability.`;
|
||||
} else if (error instanceof Error) {
|
||||
message = process.env.NODE_ENV === 'production' ? error.message : error.stack;
|
||||
}
|
||||
|
||||
logger.error(`Error in ${ctx.request.path}: ${message}`);
|
||||
|
||||
if (config.isPackage) {
|
||||
ctx.body = {
|
||||
error: {
|
||||
message: error.message ?? error,
|
||||
},
|
||||
};
|
||||
} else {
|
||||
ctx.set({
|
||||
'Content-Type': 'text/html; charset=UTF-8',
|
||||
});
|
||||
|
||||
if (error instanceof RequestInProgressError) {
|
||||
ctx.status = 503;
|
||||
message = error.message;
|
||||
ctx.set('Cache-Control', `public, max-age=${config.cache.requestTimeout}`);
|
||||
} else if (ctx.status === 403) {
|
||||
message = error.message;
|
||||
} else {
|
||||
ctx.status = 404;
|
||||
}
|
||||
|
||||
const requestPath = ctx.request.path;
|
||||
|
||||
ctx.body = art(path.resolve(__dirname, '../views/error.art'), {
|
||||
requestPath,
|
||||
message,
|
||||
errorPath: ctx.path,
|
||||
nodeVersion: process.version,
|
||||
gitHash,
|
||||
});
|
||||
}
|
||||
|
||||
if (!ctx.debug.errorPaths[ctx.request.path]) {
|
||||
ctx.debug.errorPaths[ctx.request.path] = 0;
|
||||
}
|
||||
ctx.debug.errorPaths[ctx.request.path]++;
|
||||
|
||||
if (!ctx.debug.errorRoutes[ctx._matchedRoute]) {
|
||||
ctx._matchedRoute && (ctx.debug.errorRoutes[ctx._matchedRoute] = 0);
|
||||
}
|
||||
ctx._matchedRoute && ctx.debug.errorRoutes[ctx._matchedRoute]++;
|
||||
|
||||
if (!ctx.state.debuged) {
|
||||
if (!ctx.debug.routes[ctx._matchedRoute]) {
|
||||
ctx._matchedRoute && (ctx.debug.routes[ctx._matchedRoute] = 0);
|
||||
}
|
||||
ctx._matchedRoute && ctx.debug.routes[ctx._matchedRoute]++;
|
||||
|
||||
if (ctx.response.get('X-Koa-Redis-Cache') || ctx.response.get('X-Koa-Memory-Cache')) {
|
||||
ctx.debug.hitCache++;
|
||||
}
|
||||
}
|
||||
|
||||
if (config.sentry.dsn) {
|
||||
Sentry.withScope((scope) => {
|
||||
scope.setTag('route', ctx._matchedRoute);
|
||||
scope.setTag('name', ctx.request.path.split('/')[1]);
|
||||
scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, ctx.request));
|
||||
Sentry.captureException(error);
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,92 @@
|
||||
import { MiddlewareHandler } from "hono";
|
||||
import logger from "@/utils/logger";
|
||||
import { config } from "@/config";
|
||||
import art from 'art-template';
|
||||
import * as path from 'node:path';
|
||||
import { RequestInProgressError } from '@/errors';
|
||||
import Sentry from '@sentry/node';
|
||||
import { getRouteNameFromPath } from "@/utils/helpers";
|
||||
import gitRevSync from 'git-rev-sync';
|
||||
|
||||
let gitHash;
|
||||
|
||||
if (config.sentry.dsn) {
|
||||
Sentry.init({
|
||||
dsn: config.sentry.dsn,
|
||||
});
|
||||
Sentry.getCurrentScope().setTag('node_name', config.nodeName);
|
||||
|
||||
logger.info('Sentry inited.');
|
||||
}
|
||||
|
||||
try {
|
||||
gitHash = gitRevSync.short();
|
||||
} catch {
|
||||
gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GIT_COMMIT_SHA && process.env.VERCEL_GIT_COMMIT_SHA.slice(0, 7)) || 'unknown';
|
||||
}
|
||||
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
try {
|
||||
const time = Date.now();
|
||||
await next();
|
||||
if (config.sentry.dsn && Date.now() - time >= config.sentry.routeTimeout) {
|
||||
Sentry.withScope((scope) => {
|
||||
scope.setTag('name', getRouteNameFromPath(ctx.req.path));
|
||||
Sentry.captureException(new Error('Route Timeout'));
|
||||
});
|
||||
}
|
||||
} catch (error: any) {
|
||||
let message = error;
|
||||
if (error.name && (error.name === 'HTTPError' || error.name === 'RequestError')) {
|
||||
message = `${error.message}: target website might be blocking our access, you can <a href="https://docs.rsshub.app/install/">host your own RSSHub instance</a> for a better usability.`;
|
||||
} else if (error instanceof Error) {
|
||||
message = process.env.NODE_ENV === 'production' ? error.message : error.stack;
|
||||
}
|
||||
|
||||
logger.error(`Error in ${ctx.req.path}: ${message}`);
|
||||
|
||||
if (config.isPackage) {
|
||||
ctx.json({
|
||||
error: {
|
||||
message: error.message ?? error,
|
||||
},
|
||||
});
|
||||
} else {
|
||||
ctx.header('Content-Type', 'text/html; charset=UTF-8');
|
||||
|
||||
if (error instanceof RequestInProgressError) {
|
||||
ctx.status(503);
|
||||
message = error.message;
|
||||
ctx.set('Cache-Control', `public, max-age=${config.cache.requestTimeout}`);
|
||||
} else if (ctx.res.status === 403) {
|
||||
message = error.message;
|
||||
} else {
|
||||
ctx.status(404);
|
||||
}
|
||||
|
||||
const requestPath = ctx.req.path;
|
||||
|
||||
ctx.body = art(path.resolve(__dirname, '../views/error.art'), {
|
||||
requestPath,
|
||||
message,
|
||||
errorPath: ctx.req.path,
|
||||
nodeVersion: process.version,
|
||||
gitHash,
|
||||
});
|
||||
}
|
||||
|
||||
const debug = ctx.get('debug');
|
||||
if (ctx.res.headers.get('X-Koa-Redis-Cache') || ctx.res.headers.get('X-Koa-Memory-Cache')) {
|
||||
debug.hitCache++;
|
||||
}
|
||||
|
||||
if (config.sentry.dsn) {
|
||||
Sentry.withScope((scope) => {
|
||||
scope.setTag('name', ctx.req.path.split('/')[1]);
|
||||
Sentry.captureException(error);
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
@@ -1,370 +0,0 @@
|
||||
const entities = require('entities');
|
||||
const cheerio = require('cheerio');
|
||||
const { simplecc } = require('simplecc-wasm');
|
||||
const got = require('@/utils/got');
|
||||
const config = require('@/config').value;
|
||||
const { RE2JS } = require('re2js');
|
||||
const md = require('markdown-it')({
|
||||
html: true,
|
||||
});
|
||||
const htmlToText = require('html-to-text');
|
||||
|
||||
let mercury_parser;
|
||||
|
||||
const resolveRelativeLink = ($, elem, attr, baseUrl) => {
|
||||
const $elem = $(elem);
|
||||
|
||||
if (baseUrl) {
|
||||
try {
|
||||
const oldAttr = $elem.attr(attr);
|
||||
if (oldAttr) {
|
||||
// e.g. <video><source src="https://example.com"></video> should leave <video> unchanged
|
||||
$elem.attr(attr, new URL(oldAttr, baseUrl).href);
|
||||
}
|
||||
} catch {
|
||||
// no-empty
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const summarizeArticle = async (articleText) => {
|
||||
const apiUrl = `${config.openai.endpoint}/chat/completions`;
|
||||
const response = await got.post(apiUrl, {
|
||||
json: {
|
||||
model: config.openai.model,
|
||||
max_tokens: config.openai.maxTokens,
|
||||
messages: [
|
||||
{ role: 'system', content: config.openai.prompt },
|
||||
{ role: 'user', content: articleText },
|
||||
],
|
||||
temperature: config.openai.temperature,
|
||||
},
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.openai.apiKey}`,
|
||||
},
|
||||
});
|
||||
|
||||
return response.data.choices[0].message.content;
|
||||
};
|
||||
|
||||
module.exports = async (ctx, next) => {
|
||||
await next();
|
||||
|
||||
if (!ctx.state.data && !ctx._matchedRoute) {
|
||||
throw new Error('wrong path');
|
||||
}
|
||||
|
||||
if (ctx.state.data) {
|
||||
if ((!ctx.state.data.item || ctx.state.data.item.length === 0) && !ctx.state.data.allowEmpty) {
|
||||
throw new Error('this route is empty, please check the original site or <a href="https://github.com/DIYgod/RSSHub/issues/new/choose">create an issue</a>');
|
||||
}
|
||||
|
||||
// fix allowEmpty
|
||||
ctx.state.data.item = ctx.state.data.item || [];
|
||||
|
||||
// decode HTML entities
|
||||
ctx.state.data.title && (ctx.state.data.title = entities.decodeXML(ctx.state.data.title + ''));
|
||||
ctx.state.data.description && (ctx.state.data.description = entities.decodeXML(ctx.state.data.description + ''));
|
||||
|
||||
// sort items
|
||||
if (ctx.query.sorted !== 'false') {
|
||||
ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
|
||||
}
|
||||
|
||||
const handleItem = (item) => {
|
||||
item.title && (item.title = entities.decodeXML(item.title + ''));
|
||||
|
||||
// handle pubDate
|
||||
if (item.pubDate) {
|
||||
item.pubDate = new Date(item.pubDate).toUTCString();
|
||||
}
|
||||
|
||||
// handle link
|
||||
if (item.link) {
|
||||
let baseUrl = ctx.state.data.link;
|
||||
if (baseUrl && !/^https?:\/\//.test(baseUrl)) {
|
||||
baseUrl = /^\/\//.test(baseUrl) ? 'http:' + baseUrl : 'http://' + baseUrl;
|
||||
}
|
||||
|
||||
item.link = new URL(item.link, baseUrl).href;
|
||||
}
|
||||
|
||||
// handle description
|
||||
if (item.description) {
|
||||
const $ = cheerio.load(item.description);
|
||||
let baseUrl = item.link || ctx.state.data.link;
|
||||
|
||||
if (baseUrl && !/^https?:\/\//.test(baseUrl)) {
|
||||
baseUrl = /^\/\//.test(baseUrl) ? 'http:' + baseUrl : 'http://' + baseUrl;
|
||||
}
|
||||
|
||||
$('script').remove();
|
||||
|
||||
$('img').each((_, ele) => {
|
||||
const $ele = $(ele);
|
||||
|
||||
// fix lazyload
|
||||
if (!$ele.attr('src')) {
|
||||
const lazySrc = $ele.attr('data-src') || $ele.attr('data-original');
|
||||
if (lazySrc) {
|
||||
$ele.attr('src', lazySrc);
|
||||
} else {
|
||||
for (const key in ele.attribs) {
|
||||
const value = ele.attribs[key].trim();
|
||||
if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
|
||||
$ele.attr('src', value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// redundant attributes
|
||||
for (const e of ['onclick', 'onerror', 'onload']) {
|
||||
$ele.removeAttr(e);
|
||||
}
|
||||
});
|
||||
|
||||
// resolve relative link & fix referrer policy
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
|
||||
// https://www.w3schools.com/tags/att_href.asp
|
||||
$('a, area').each((_, elem) => {
|
||||
resolveRelativeLink($, elem, 'href', baseUrl);
|
||||
// $(elem).attr('rel', 'noreferrer'); // currently no such a need
|
||||
});
|
||||
// https://www.w3schools.com/tags/att_src.asp
|
||||
$('img, video, audio, source, iframe, embed, track').each((_, elem) => {
|
||||
resolveRelativeLink($, elem, 'src', baseUrl);
|
||||
});
|
||||
$('video[poster]').each((_, elem) => {
|
||||
resolveRelativeLink($, elem, 'poster', baseUrl);
|
||||
});
|
||||
$('img, iframe').each((_, elem) => {
|
||||
$(elem).attr('referrerpolicy', 'no-referrer');
|
||||
});
|
||||
|
||||
item.description = $('body').html() + '' + (config.suffix || '');
|
||||
|
||||
if (item._extra?.links && $('.rsshub-quote').length) {
|
||||
item._extra?.links?.map((e) => {
|
||||
e.content_html = $.html($('.rsshub-quote'));
|
||||
return e;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// handle category
|
||||
if (item.category) {
|
||||
// convert single string to array, and filter only string type category
|
||||
Array.isArray(item.category) || (item.category = [item.category]);
|
||||
item.category = item.category.filter((e) => typeof e === 'string');
|
||||
}
|
||||
return item;
|
||||
};
|
||||
|
||||
ctx.state.data.item = await Promise.all(ctx.state.data.item.map((itm) => handleItem(itm)));
|
||||
|
||||
if (ctx.query) {
|
||||
// filter
|
||||
const engine = config.feature.filter_regex_engine;
|
||||
const makeRegex = (string) => {
|
||||
if (!string) {
|
||||
return null;
|
||||
}
|
||||
// default: case_senstivie = true
|
||||
const insensitive = ctx.query.filter_case_sensitive === 'false';
|
||||
switch (engine) {
|
||||
case 'regexp':
|
||||
return new RegExp(string, insensitive ? 'i' : '');
|
||||
case 're2':
|
||||
return RE2JS.compile(string, insensitive ? RE2JS.CASE_INSENSITIVE : 0);
|
||||
default:
|
||||
throw new Error(`Invalid Engine Value: ${engine}, please check your config.`);
|
||||
}
|
||||
};
|
||||
|
||||
if (ctx.query.filter) {
|
||||
const regex = makeRegex(ctx.query.filter);
|
||||
|
||||
ctx.state.data.item = ctx.state.data.item.filter((item) => {
|
||||
const title = item.title || '';
|
||||
const description = item.description || title;
|
||||
const author = item.author || '';
|
||||
const category = item.category || [];
|
||||
const isFilter =
|
||||
engine === 're2'
|
||||
? regex.matcher(title).find() || regex.matcher(description).find() || regex.matcher(author).find() || category.some((c) => regex.matcher(c).find())
|
||||
: title.match(regex) || description.match(regex) || author.match(regex) || category.some((c) => c.match(regex));
|
||||
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
// 启用filter参数时，无效filter_title/description/author/category
|
||||
if (!ctx.query.filter && (ctx.query.filter_title || ctx.query.filter_description || ctx.query.filter_author || ctx.query.filter_category)) {
|
||||
ctx.state.data.item = ctx.state.data.item.filter((item) => {
|
||||
const title = item.title || '';
|
||||
const description = item.description || title;
|
||||
const author = item.author || '';
|
||||
const category = item.category || [];
|
||||
let isFilter = true;
|
||||
|
||||
const titleRegex = makeRegex(ctx.query.filter_title);
|
||||
const descriptionRegex = makeRegex(ctx.query.filter_description);
|
||||
const authorRegex = makeRegex(ctx.query.filter_author);
|
||||
const categoryRegex = makeRegex(ctx.query.filter_category);
|
||||
|
||||
ctx.query.filter_title && (isFilter = engine === 're2' ? titleRegex.matcher(title).find() : title.match(titleRegex));
|
||||
ctx.query.filter_description && (isFilter = isFilter && (engine === 're2' ? descriptionRegex.matcher(description).find() : description.match(descriptionRegex)));
|
||||
ctx.query.filter_author && (isFilter = isFilter && (engine === 're2' ? authorRegex.matcher(author).find() : author.match(authorRegex)));
|
||||
ctx.query.filter_category && (isFilter = isFilter && category.some((c) => (engine === 're2' ? categoryRegex.matcher(c).find() : c.match(categoryRegex))));
|
||||
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
if (ctx.query.filterout || ctx.query.filterout_title || ctx.query.filterout_description || ctx.query.filterout_author || ctx.query.filterout_category) {
|
||||
if (ctx.query.filterout) {
|
||||
ctx.query.filterout_title = ctx.query.filterout;
|
||||
ctx.query.filterout_description = ctx.query.filterout;
|
||||
}
|
||||
ctx.state.data.item = ctx.state.data.item.filter((item) => {
|
||||
const title = item.title;
|
||||
const description = item.description || title;
|
||||
const author = item.author || '';
|
||||
const category = item.category || [];
|
||||
let isFilter = true;
|
||||
|
||||
const titleRegex = makeRegex(ctx.query.filterout_title);
|
||||
const descriptionRegex = makeRegex(ctx.query.filterout_description);
|
||||
const authorRegex = makeRegex(ctx.query.filterout_author);
|
||||
const categoryRegex = makeRegex(ctx.query.filterout_category);
|
||||
|
||||
ctx.query.filterout_title && (isFilter = engine === 're2' ? !titleRegex.matcher(title).find() : !title.match(titleRegex));
|
||||
ctx.query.filterout_description && (isFilter = isFilter && (engine === 're2' ? !descriptionRegex.matcher(description).find() : !description.match(descriptionRegex)));
|
||||
ctx.query.filterout_author && (isFilter = isFilter && (engine === 're2' ? !authorRegex.matcher(author).find() : !author.match(authorRegex)));
|
||||
ctx.query.filterout_category && (isFilter = isFilter && !category.some((c) => (engine === 're2' ? categoryRegex.matcher(c).find() : c.match(categoryRegex))));
|
||||
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
if (ctx.query.filter_time) {
|
||||
const now = Date.now();
|
||||
ctx.state.data.item = ctx.state.data.item.filter(({ pubDate }) => {
|
||||
let isFilter = true;
|
||||
try {
|
||||
isFilter = !pubDate || now - new Date(pubDate).getTime() <= Number.parseInt(ctx.query.filter_time) * 1000;
|
||||
} catch {
|
||||
// no-empty
|
||||
}
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
// limit
|
||||
if (ctx.query.limit) {
|
||||
ctx.state.data.item = ctx.state.data.item.slice(0, Number.parseInt(ctx.query.limit));
|
||||
}
|
||||
|
||||
// telegram instant view
|
||||
if (ctx.query.tgiv) {
|
||||
ctx.state.data.item.map((item) => {
|
||||
const encodedlink = encodeURIComponent(item.link);
|
||||
item.link = `https://t.me/iv?url=${encodedlink}&rhash=${ctx.query.tgiv}`;
|
||||
return item;
|
||||
});
|
||||
}
|
||||
|
||||
// fulltext
|
||||
if (ctx.query.mode && ctx.query.mode.toLowerCase() === 'fulltext') {
|
||||
const tasks = ctx.state.data.item.map(async (item) => {
|
||||
const { link, author, description } = item;
|
||||
const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
|
||||
// if parser failed, return default description and not report error
|
||||
try {
|
||||
mercury_parser = mercury_parser || require('@postlight/parser');
|
||||
|
||||
const { data: res } = await got(link);
|
||||
const $ = cheerio.load(res);
|
||||
const result = await mercury_parser.parse(link, {
|
||||
html: $.html(),
|
||||
});
|
||||
return result;
|
||||
} catch {
|
||||
// no-empty
|
||||
}
|
||||
});
|
||||
|
||||
item.author = author || (parsed_result ? parsed_result.author : '');
|
||||
item.description = parsed_result && parsed_result.content.length > 40 ? entities.decodeXML(parsed_result.content) : description;
|
||||
});
|
||||
await Promise.all(tasks);
|
||||
}
|
||||
|
||||
// openai
|
||||
if (ctx.query.chatgpt && config.openai.apiKey) {
|
||||
ctx.state.data.item = await Promise.all(
|
||||
ctx.state.data.item.map(async (item) => {
|
||||
if (item.description) {
|
||||
try {
|
||||
const summary = await ctx.cache.tryGet(`openai:${item.link}`, async () => {
|
||||
const text = htmlToText.htmlToText(item.description);
|
||||
if (text.length < 300) {
|
||||
return '';
|
||||
}
|
||||
const summary_md = await summarizeArticle(text);
|
||||
return md.render(summary_md);
|
||||
});
|
||||
// 将总结结果添加到文章数据中
|
||||
if (summary !== '') {
|
||||
item.description = summary + '<hr/><br/>' + item.description;
|
||||
}
|
||||
} catch {
|
||||
// when openai failed, return default description and not write cache
|
||||
}
|
||||
}
|
||||
return item;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// scihub
|
||||
if (ctx.query.scihub) {
|
||||
ctx.state.data.item.map((item) => {
|
||||
item.link = item.doi ? `${config.scihub.host}${item.doi}` : `${config.scihub.host}${item.link}`;
|
||||
return item;
|
||||
});
|
||||
}
|
||||
|
||||
// opencc
|
||||
if (ctx.query.opencc) {
|
||||
for (const item of ctx.state.data.item) {
|
||||
item.title = simplecc(item.title ?? item.link, ctx.query.opencc);
|
||||
item.description = simplecc(item.description ?? item.title ?? item.link, ctx.query.opencc);
|
||||
}
|
||||
}
|
||||
|
||||
// brief
|
||||
if (ctx.query.brief) {
|
||||
const num = /[1-9]\d{2,}/;
|
||||
if (num.test(ctx.query.brief)) {
|
||||
ctx.query.brief = Number.parseInt(ctx.query.brief);
|
||||
for (const item of ctx.state.data.item) {
|
||||
let text;
|
||||
if (item.description) {
|
||||
text = item.description.replaceAll(/<\/?[^>]+(>|$)/g, '');
|
||||
}
|
||||
if (text?.length) {
|
||||
item.description = text.length > ctx.query.brief ? `<p>${text.substring(0, ctx.query.brief)}…</p>` : `<p>${text}</p>`;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new Error(`Invalid parameter brief. Please check the doc https://docs.rsshub.app/parameter#shu-chu-jian-xun`);
|
||||
}
|
||||
}
|
||||
|
||||
// some parameters are processed in `anti-hotlink.js`
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,380 @@
|
||||
import * as entities from 'entities';
|
||||
import { load, type CheerioAPI, type Element } from 'cheerio';
|
||||
import { simplecc } from 'simplecc-wasm';
|
||||
import got from '@/utils/got';
|
||||
import { config } from '@/config';
|
||||
import { RE2JS } from 're2js';
|
||||
import markdownit from 'markdown-it'
|
||||
import htmlToText from 'html-to-text'
|
||||
import { MiddlewareHandler } from 'hono';
|
||||
import cache from '@/utils/cache';
|
||||
import Parser from '@postlight/parser';
|
||||
import { Data, DataItem } from '@/types';
|
||||
|
||||
const md = markdownit({
|
||||
html: true,
|
||||
})
|
||||
|
||||
const resolveRelativeLink = ($: CheerioAPI, elem: Element, attr: string, baseUrl?: string) => {
|
||||
const $elem = $(elem);
|
||||
|
||||
if (baseUrl) {
|
||||
try {
|
||||
const oldAttr = $elem.attr(attr);
|
||||
if (oldAttr) {
|
||||
// e.g. <video><source src="https://example.com"></video> should leave <video> unchanged
|
||||
$elem.attr(attr, new URL(oldAttr, baseUrl).href);
|
||||
}
|
||||
} catch {
|
||||
// no-empty
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const summarizeArticle = async (articleText: string) => {
|
||||
const apiUrl = `${config.openai.endpoint}/chat/completions`;
|
||||
const response = await got.post(apiUrl, {
|
||||
json: {
|
||||
model: config.openai.model,
|
||||
max_tokens: config.openai.maxTokens,
|
||||
messages: [
|
||||
{ role: 'system', content: config.openai.prompt },
|
||||
{ role: 'user', content: articleText },
|
||||
],
|
||||
temperature: config.openai.temperature,
|
||||
},
|
||||
headers: {
|
||||
Authorization: `Bearer ${config.openai.apiKey}`,
|
||||
},
|
||||
});
|
||||
|
||||
return response.data.choices[0].message.content;
|
||||
};
|
||||
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
await next();
|
||||
|
||||
const data = ctx.get('data') as Data;
|
||||
if (!data) {
|
||||
throw new Error('wrong path');
|
||||
} else {
|
||||
if ((!data.item || data.item.length === 0) && !data.allowEmpty) {
|
||||
throw new Error('this route is empty, please check the original site or <a href="https://github.com/DIYgod/RSSHub/issues/new/choose">create an issue</a>');
|
||||
}
|
||||
|
||||
// fix allowEmpty
|
||||
data.item = data.item || [];
|
||||
|
||||
// decode HTML entities
|
||||
data.title && (data.title = entities.decodeXML(data.title + ''));
|
||||
data.description && (data.description = entities.decodeXML(data.description + ''));
|
||||
|
||||
// sort items
|
||||
if (ctx.req.query('sorted') !== 'false') {
|
||||
data.item = data.item.sort((a: DataItem, b: DataItem) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
|
||||
}
|
||||
|
||||
const handleItem = (item: DataItem) => {
|
||||
item.title && (item.title = entities.decodeXML(item.title + ''));
|
||||
|
||||
// handle pubDate
|
||||
if (item.pubDate) {
|
||||
item.pubDate = new Date(item.pubDate).toUTCString();
|
||||
}
|
||||
|
||||
// handle link
|
||||
if (item.link) {
|
||||
let baseUrl = data.link;
|
||||
if (baseUrl && !/^https?:\/\//.test(baseUrl)) {
|
||||
baseUrl = /^\/\//.test(baseUrl) ? 'http:' + baseUrl : 'http://' + baseUrl;
|
||||
}
|
||||
|
||||
item.link = new URL(item.link, baseUrl).href;
|
||||
}
|
||||
|
||||
// handle description
|
||||
if (item.description) {
|
||||
const $ = load(item.description);
|
||||
let baseUrl = item.link || data.link;
|
||||
|
||||
if (baseUrl && !/^https?:\/\//.test(baseUrl)) {
|
||||
baseUrl = /^\/\//.test(baseUrl) ? 'http:' + baseUrl : 'http://' + baseUrl;
|
||||
}
|
||||
|
||||
$('script').remove();
|
||||
|
||||
$('img').each((_, ele) => {
|
||||
const $ele = $(ele);
|
||||
|
||||
// fix lazyload
|
||||
if (!$ele.attr('src')) {
|
||||
const lazySrc = $ele.attr('data-src') || $ele.attr('data-original');
|
||||
if (lazySrc) {
|
||||
$ele.attr('src', lazySrc);
|
||||
} else {
|
||||
for (const key in ele.attribs) {
|
||||
const value = ele.attribs[key].trim();
|
||||
if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
|
||||
$ele.attr('src', value);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// redundant attributes
|
||||
for (const e of ['onclick', 'onerror', 'onload']) {
|
||||
$ele.removeAttr(e);
|
||||
}
|
||||
});
|
||||
|
||||
// resolve relative link & fix referrer policy
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
|
||||
// https://www.w3schools.com/tags/att_href.asp
|
||||
$('a, area').each((_, elem) => {
|
||||
resolveRelativeLink($, elem, 'href', baseUrl);
|
||||
// $(elem).attr('rel', 'noreferrer'); // currently no such a need
|
||||
});
|
||||
// https://www.w3schools.com/tags/att_src.asp
|
||||
$('img, video, audio, source, iframe, embed, track').each((_, elem) => {
|
||||
resolveRelativeLink($, elem, 'src', baseUrl);
|
||||
});
|
||||
$('video[poster]').each((_, elem) => {
|
||||
resolveRelativeLink($, elem, 'poster', baseUrl);
|
||||
});
|
||||
$('img, iframe').each((_, elem) => {
|
||||
$(elem).attr('referrerpolicy', 'no-referrer');
|
||||
});
|
||||
|
||||
item.description = $('body').html() + '' + (config.suffix || '');
|
||||
|
||||
if (item._extra?.links && $('.rsshub-quote').length) {
|
||||
item._extra?.links?.map((e) => {
|
||||
e.content_html = $.html($('.rsshub-quote'));
|
||||
return e;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// handle category
|
||||
if (item.category) {
|
||||
// convert single string to array, and filter only string type category
|
||||
Array.isArray(item.category) || (item.category = [item.category]);
|
||||
item.category = item.category.filter((e) => typeof e === 'string');
|
||||
}
|
||||
return item;
|
||||
};
|
||||
|
||||
data.item = await Promise.all(data.item.map((itm) => handleItem(itm)));
|
||||
|
||||
// filter
|
||||
const engine = config.feature.filter_regex_engine;
|
||||
const makeRegex = (str: string) => {
|
||||
// default: case_senstivie = true
|
||||
const insensitive = ctx.req.query('filter_case_sensitive') === 'false';
|
||||
switch (engine) {
|
||||
case 'regexp':
|
||||
return new RegExp(str, insensitive ? 'i' : '');
|
||||
case 're2':
|
||||
return RE2JS.compile(str, insensitive ? RE2JS.CASE_INSENSITIVE : 0);
|
||||
default:
|
||||
throw new Error(`Invalid Engine Value: ${engine}, please check your config.`);
|
||||
}
|
||||
};
|
||||
|
||||
if (ctx.req.query('filter')) {
|
||||
const regex = makeRegex(ctx.req.query('filter')!);
|
||||
|
||||
data.item = data.item.filter((item) => {
|
||||
const title = item.title || '';
|
||||
const description = item.description || title;
|
||||
const author = item.author || '';
|
||||
const category = item.category || [];
|
||||
const isFilter =
|
||||
regex instanceof RE2JS
|
||||
? regex.matcher(title).find() || regex.matcher(description).find() || regex.matcher(author).find() || category.some((c) => regex.matcher(c).find())
|
||||
: title.match(regex) || description.match(regex) || author.match(regex) || category.some((c) => c.match(regex));
|
||||
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
// 启用filter参数时，无效filter_title/description/author/category
|
||||
if (!ctx.req.query('filter') && (ctx.req.query('filter_title') || ctx.req.query('filter_description') || ctx.req.query('filter_author') || ctx.req.query('filter_category'))) {
|
||||
data.item = data.item.filter((item) => {
|
||||
const title = item.title || '';
|
||||
const description = item.description || title;
|
||||
const author = item.author || '';
|
||||
const category = item.category || [];
|
||||
let isFilter = true;
|
||||
|
||||
if (ctx.req.query('filter_title')) {
|
||||
const titleRegex = makeRegex(ctx.req.query('filter_title')!);
|
||||
isFilter = titleRegex instanceof RE2JS ? titleRegex.matcher(title).find() : !!title.match(titleRegex);
|
||||
}
|
||||
if (ctx.req.query('filter_description')) {
|
||||
const descriptionRegex = makeRegex(ctx.req.query('filter_description')!);
|
||||
isFilter = isFilter && (descriptionRegex instanceof RE2JS ? descriptionRegex.matcher(description).find() : !!description.match(descriptionRegex));
|
||||
}
|
||||
if (ctx.req.query('filter_author')) {
|
||||
const authorRegex = makeRegex(ctx.req.query('filter_author')!);
|
||||
isFilter = isFilter && (authorRegex instanceof RE2JS ? authorRegex.matcher(author).find() : !!author.match(authorRegex));
|
||||
}
|
||||
if (ctx.req.query('filter_category')) {
|
||||
const categoryRegex = makeRegex(ctx.req.query('filter_category')!);
|
||||
isFilter = isFilter && category.some((c) => (categoryRegex instanceof RE2JS ? categoryRegex.matcher(c).find() : c.match(categoryRegex)));
|
||||
}
|
||||
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
if (ctx.req.query('filterout') || ctx.req.query('filterout_title') || ctx.req.query('filterout_description') || ctx.req.query('filterout_author') || ctx.req.query('filterout_category')) {
|
||||
data.item = data.item.filter((item) => {
|
||||
const title = item.title;
|
||||
const description = item.description || title;
|
||||
const author = item.author || '';
|
||||
const category = item.category || [];
|
||||
let isFilter = true;
|
||||
|
||||
if (ctx.req.query('filterout')) {
|
||||
const titleRegex = makeRegex(ctx.req.query('filterout_title') || ctx.req.query('filterout')!);
|
||||
const descriptionRegex = makeRegex(ctx.req.query('filterout_description') || ctx.req.query('filterout')!);
|
||||
|
||||
isFilter = titleRegex instanceof RE2JS ? !titleRegex.matcher(title).find() : !title.match(titleRegex);
|
||||
isFilter = isFilter && (descriptionRegex instanceof RE2JS ? !descriptionRegex.matcher(description).find() : !description.match(descriptionRegex));
|
||||
}
|
||||
if (ctx.req.query('filterout_author')) {
|
||||
const authorRegex = makeRegex(ctx.req.query('filterout_author')!);
|
||||
isFilter = isFilter && (authorRegex instanceof RE2JS ? !authorRegex.matcher(author).find() : !author.match(authorRegex));
|
||||
}
|
||||
if (ctx.req.query('filterout_category')) {
|
||||
const categoryRegex = makeRegex(ctx.req.query('filterout_category')!);
|
||||
isFilter = isFilter && !category.some((c) => (categoryRegex instanceof RE2JS ? categoryRegex.matcher(c).find() : c.match(categoryRegex)));
|
||||
}
|
||||
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
if (ctx.req.query('filter_time')) {
|
||||
const now = Date.now();
|
||||
data.item = data.item.filter(({ pubDate }) => {
|
||||
let isFilter = true;
|
||||
try {
|
||||
isFilter = !pubDate || now - new Date(pubDate).getTime() <= Number.parseInt(ctx.req.query('filter_time')!) * 1000;
|
||||
} catch {
|
||||
// no-empty
|
||||
}
|
||||
return isFilter;
|
||||
});
|
||||
}
|
||||
|
||||
// limit
|
||||
if (ctx.req.query('limit')) {
|
||||
data.item = data.item.slice(0, Number.parseInt(ctx.req.query('limit')!));
|
||||
}
|
||||
|
||||
// telegram instant view
|
||||
if (ctx.req.query('tgiv')) {
|
||||
data.item.map((item) => {
|
||||
if (item.link) {
|
||||
const encodedlink = encodeURIComponent(item.link);
|
||||
item.link = `https://t.me/iv?url=${encodedlink}&rhash=${ctx.req.query('tgiv')}`;
|
||||
return item;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// fulltext
|
||||
if (ctx.req.query('mode')?.toLowerCase() === 'fulltext') {
|
||||
const tasks = data.item.map(async (item) => {
|
||||
const { link, author, description } = item;
|
||||
const parsed_result: any = await cache.tryGet(`mercury-cache-${link}`, async () => {
|
||||
if (link) {
|
||||
// if parser failed, return default description and not report error
|
||||
try {
|
||||
const { data: res } = await got(link);
|
||||
const $ = load(res);
|
||||
const result = await Parser.parse(link, {
|
||||
html: $.html(),
|
||||
});
|
||||
return result;
|
||||
} catch {
|
||||
// no-empty
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
item.author = author || parsed_result?.author;
|
||||
item.description = parsed_result && parsed_result.content.length > 40 ? entities.decodeXML(parsed_result.content) : description;
|
||||
});
|
||||
await Promise.all(tasks);
|
||||
}
|
||||
|
||||
// openai
|
||||
if (ctx.req.query('chatgpt') && config.openai.apiKey) {
|
||||
data.item = await Promise.all(
|
||||
data.item.map(async (item) => {
|
||||
if (item.description) {
|
||||
try {
|
||||
const summary = await cache.tryGet(`openai:${item.link}`, async () => {
|
||||
const text = htmlToText.htmlToText(item.description!);
|
||||
if (text.length < 300) {
|
||||
return '';
|
||||
}
|
||||
const summary_md = await summarizeArticle(text);
|
||||
return md.render(summary_md);
|
||||
});
|
||||
// 将总结结果添加到文章数据中
|
||||
if (summary !== '') {
|
||||
item.description = summary + '<hr/><br/>' + item.description;
|
||||
}
|
||||
} catch {
|
||||
// when openai failed, return default description and not write cache
|
||||
}
|
||||
}
|
||||
return item;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// scihub
|
||||
if (ctx.req.query('scihub')) {
|
||||
data.item.map((item) => {
|
||||
item.link = item.doi ? `${config.scihub.host}${item.doi}` : `${config.scihub.host}${item.link}`;
|
||||
return item;
|
||||
});
|
||||
}
|
||||
|
||||
// opencc
|
||||
if (ctx.req.query('opencc')) {
|
||||
for (const item of data.item) {
|
||||
item.title = simplecc(item.title ?? item.link, ctx.req.query('opencc')!);
|
||||
item.description = simplecc(item.description ?? item.title ?? item.link, ctx.req.query('opencc')!);
|
||||
}
|
||||
}
|
||||
|
||||
// brief
|
||||
if (ctx.req.query('brief')) {
|
||||
const num = /[1-9]\d{2,}/;
|
||||
if (num.test(ctx.req.query('brief')!)) {
|
||||
const brief = Number.parseInt(ctx.req.query('brief')!);
|
||||
for (const item of data.item) {
|
||||
let text;
|
||||
if (item.description) {
|
||||
text = item.description.replaceAll(/<\/?[^>]+(>|$)/g, '');
|
||||
item.description = text.length > brief ? `<p>${text.substring(0, brief)}…</p>` : `<p>${text}</p>`;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
throw new Error(`Invalid parameter brief. Please check the doc https://docs.rsshub.app/parameter#shu-chu-jian-xun`);
|
||||
}
|
||||
}
|
||||
// some parameters are processed in `anti-hotlink.js`
|
||||
|
||||
ctx.set('data', data)
|
||||
}
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
@@ -1,59 +1,51 @@
|
||||
const { art, json, rss3Ums } = require('@/utils/render');
|
||||
const path = require('path');
|
||||
const config = require('@/config').value;
|
||||
import render from '@/utils/render';
|
||||
import * as path from 'node:path';
|
||||
import { config } from '@/config';
|
||||
const typeRegex = /\.(atom|rss|ums|debug\.json|json|\d+\.debug\.html)$/;
|
||||
const { collapseWhitespace, convertDateToISO8601 } = require('@/utils/common-utils');
|
||||
import utils from '@/utils/common-utils';
|
||||
import type { MiddlewareHandler } from 'hono';
|
||||
|
||||
module.exports = async (ctx, next) => {
|
||||
if (ctx.headers['user-agent'] && ctx.headers['user-agent'].includes('Reeder')) {
|
||||
ctx.request.path = ctx.request.path.replace(/.com$/, '');
|
||||
const middleware: MiddlewareHandler = async (ctx, next) => {
|
||||
if (ctx.req.header('user-agent')?.includes('Reeder')) {
|
||||
ctx.req.path = ctx.req.path.replace(/.com$/, '');
|
||||
}
|
||||
|
||||
ctx.state.type = ctx.request.path.match(typeRegex) || ['', ''];
|
||||
ctx.request.path = ctx.request.path.replace(typeRegex, '');
|
||||
ctx.set('type', ctx.req.path.match(typeRegex) || ['', '']);
|
||||
ctx.req.path = ctx.req.path.replace(typeRegex, '');
|
||||
|
||||
await next();
|
||||
|
||||
const outputType = ctx.state.type[1] || 'rss';
|
||||
const outputType = ctx.get('type')[1] || 'rss';
|
||||
|
||||
// only enable when debugInfo=true
|
||||
if (config.debugInfo) {
|
||||
if (outputType === 'debug.json') {
|
||||
ctx.set({
|
||||
'Content-Type': 'application/json; charset=UTF-8',
|
||||
});
|
||||
ctx.body = ctx.state.json ? JSON.stringify(ctx.state.json, null, 4) : JSON.stringify({ message: 'plugin does not set debug json' });
|
||||
ctx.header('Content-Type', 'application/json; charset=UTF-8');
|
||||
return ctx.body(ctx.get('json') ? JSON.stringify(ctx.get('json'), null, 4) : JSON.stringify({ message: 'plugin does not set debug json' }));
|
||||
}
|
||||
|
||||
if (outputType.endsWith('.debug.html')) {
|
||||
ctx.set({
|
||||
'Content-Type': 'text/html; charset=UTF-8',
|
||||
});
|
||||
ctx.header('Content-Type', 'text/html; charset=UTF-8');
|
||||
|
||||
const index = Number.parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]);
|
||||
ctx.body = ctx.state.data && ctx.state.data.item && ctx.state.data.item[index] ? ctx.state.data.item[index].description : `ctx.state.data.item[${index}] not found`;
|
||||
return ctx.body(ctx.get('data')?.item?.[index]?.description || `ctx.get('data')?.item?.[${index}]?.description not found`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!ctx.body) {
|
||||
const templateName = outputType === 'atom' ? 'atom.art' : 'rss.art';
|
||||
const template = path.resolve(__dirname, `../views/${templateName}`);
|
||||
|
||||
if (ctx.state.data) {
|
||||
const collapseWhitespaceForProperties = (properties, obj) => {
|
||||
for (const prop of properties) {
|
||||
if (obj[prop]) {
|
||||
obj[prop] = collapseWhitespace(obj[prop]);
|
||||
}
|
||||
if (ctx.get('data')) {
|
||||
for (const prop of ['title', 'subtitle', 'author']) {
|
||||
if (ctx.get('data')[prop]) {
|
||||
ctx.get('data')[prop] = utils.collapseWhitespace(ctx.get('data')[prop]);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
collapseWhitespaceForProperties(['title', 'subtitle', 'author'], ctx.state.data);
|
||||
|
||||
if (ctx.state.data.item) {
|
||||
for (const item of ctx.state.data.item) {
|
||||
if (ctx.get('data').item) {
|
||||
for (const item of ctx.get('data').item) {
|
||||
if (item.title) {
|
||||
item.title = collapseWhitespace(item.title);
|
||||
item.title = utils.collapseWhitespace(item.title);
|
||||
// trim title length
|
||||
for (let length = 0, i = 0; i < item.title.length; i++) {
|
||||
length += Buffer.from(item.title[i]).length === 1 ? 1 : 2;
|
||||
@@ -65,13 +57,15 @@ module.exports = async (ctx, next) => {
|
||||
}
|
||||
|
||||
if (typeof item.author === 'string') {
|
||||
item.author = collapseWhitespace(item.author);
|
||||
item.author = utils.collapseWhitespace(item.author);
|
||||
} else if (typeof item.author === 'object' && item.author !== null) {
|
||||
for (const a of item.author) {
|
||||
a.name = collapseWhitespace(a.name);
|
||||
a.name = utils.collapseWhitespace(a.name);
|
||||
}
|
||||
if (outputType !== 'json') {
|
||||
item.author = item.author.map((a) => a.name).join(', ');
|
||||
item.author = item.author.map((a: {
|
||||
name: string;
|
||||
}) => a.name).join(', ');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,8 +76,8 @@ module.exports = async (ctx, next) => {
|
||||
}
|
||||
|
||||
if (outputType !== 'rss') {
|
||||
item.pubDate = convertDateToISO8601(item.pubDate);
|
||||
item.updated = convertDateToISO8601(item.updated);
|
||||
item.pubDate = utils.convertDateToISO8601(item.pubDate);
|
||||
item.updated = utils.convertDateToISO8601(item.updated);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -94,23 +88,23 @@ module.exports = async (ctx, next) => {
|
||||
lastBuildDate: currentDate.toUTCString(),
|
||||
updated: currentDate.toISOString(),
|
||||
ttl: Math.trunc(config.cache.routeExpire / 60),
|
||||
atomlink: ctx.request.href,
|
||||
...ctx.state.data,
|
||||
atomlink: ctx.req.url,
|
||||
...ctx.get('data'),
|
||||
};
|
||||
|
||||
if (config.isPackage) {
|
||||
ctx.body = data;
|
||||
return;
|
||||
return ctx.body(data);
|
||||
}
|
||||
|
||||
if (outputType === 'ums') {
|
||||
ctx.set({ 'Content-Type': 'application/json; charset=UTF-8' });
|
||||
ctx.body = rss3Ums(data);
|
||||
ctx.header('Content-Type', 'application/json; charset=UTF-8');
|
||||
return ctx.body(render.rss3Ums(data));
|
||||
} else if (outputType === 'json') {
|
||||
ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' });
|
||||
ctx.body = json(data);
|
||||
ctx.header('Content-Type', 'application/feed+json; charset=UTF-8');
|
||||
return ctx.body(render.json(data));
|
||||
} else {
|
||||
ctx.body = art(template, data);
|
||||
return ctx.body(render.art(template, data));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export default middleware;
|
||||
@@ -0,0 +1,21 @@
|
||||
import { directoryImport } from '@/utils/directory-import';
|
||||
import type { Context, Handler, Hono } from 'hono'
|
||||
|
||||
type Root = {
|
||||
get: (path: string, handler: Handler) => void;
|
||||
}
|
||||
|
||||
const imports = directoryImport({
|
||||
callerDirectoryPath: __dirname,
|
||||
targetDirectoryPath: './v3',
|
||||
importPattern: /router\.js$/,
|
||||
});
|
||||
|
||||
const routes: Record<string, (root: Root) => void> = {};
|
||||
|
||||
for (const path in imports) {
|
||||
const name = path.split('/').filter(Boolean)[0];
|
||||
routes[name] = imports[path] as (root: Root) => void;
|
||||
}
|
||||
|
||||
export default routes;
|
||||
@@ -0,0 +1,24 @@
|
||||
export type DataItem = {
|
||||
title: string;
|
||||
description?: string;
|
||||
pubDate: number | string;
|
||||
link?: string;
|
||||
category?: string[];
|
||||
author?: string;
|
||||
doi?: string;
|
||||
|
||||
_extra?: Record<string, any> & {
|
||||
links?: {
|
||||
url: string;
|
||||
type: string;
|
||||
content_html?: string;
|
||||
}[];
|
||||
};
|
||||
}
|
||||
export type Data = {
|
||||
title: string;
|
||||
description: string;
|
||||
link?: string;
|
||||
item: DataItem[];
|
||||
allowEmpty?: boolean;
|
||||
}
|
||||
Vendored
+16
@@ -0,0 +1,16 @@
|
||||
import type Redis from 'ioredis';
|
||||
import type { LRUCache } from 'lru-cache';
|
||||
|
||||
type CacheModule = {
|
||||
get: (key: string, refresh?: boolean) => Promise<string | null> | string | null;
|
||||
set: (key: string, value?: string | Record<string, any>, maxAge?: number) => any;
|
||||
status: {
|
||||
available: boolean;
|
||||
};
|
||||
clients: {
|
||||
redisClient?: Redis;
|
||||
memoryCache?: LRUCache<{}, {}>;
|
||||
}
|
||||
};
|
||||
|
||||
export default CacheModule;
|
||||
Vendored
+77
@@ -0,0 +1,77 @@
|
||||
import { config } from '@/config';
|
||||
import redis from './redis';
|
||||
import memory from './memory';
|
||||
import type CacheModule from './base';
|
||||
|
||||
const globalCache: {
|
||||
get: (key: string) => Promise<string | null | undefined> | string | null | undefined;
|
||||
set: (key: string, value?: string | Record<string, any>, maxAge?: number) => any;
|
||||
} = {
|
||||
get: () => null,
|
||||
set: () => null,
|
||||
};
|
||||
|
||||
let cacheModule: CacheModule
|
||||
|
||||
if (config.cache.type === 'redis') {
|
||||
cacheModule = redis;
|
||||
const { redisClient } = cacheModule.clients;
|
||||
globalCache.get = async (key) => {
|
||||
if (key && cacheModule.status.available && redisClient) {
|
||||
const value = await redisClient.get(key);
|
||||
return value;
|
||||
}
|
||||
};
|
||||
globalCache.set = cacheModule.set;
|
||||
} else {
|
||||
cacheModule = memory;
|
||||
const { memoryCache } = cacheModule.clients;
|
||||
globalCache.get = (key) => {
|
||||
if (key && cacheModule.status.available && memoryCache) {
|
||||
return memoryCache.get(key, { updateAgeOnGet: false }) as string | undefined;
|
||||
}
|
||||
};
|
||||
globalCache.set = (key, value, maxAge = config.cache.routeExpire) => {
|
||||
if (!value || value === 'undefined') {
|
||||
value = '';
|
||||
}
|
||||
if (typeof value === 'object') {
|
||||
value = JSON.stringify(value);
|
||||
}
|
||||
if (key && memoryCache) {
|
||||
return memoryCache.set(key, value, { ttl: maxAge * 1000 });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// only give cache string, as the `!` condition tricky
|
||||
// md5 is used to shrink key size
|
||||
// plz, write these tips in comments!
|
||||
export default {
|
||||
...cacheModule,
|
||||
tryGet: async (key: string, getValueFunc: () => Promise<string | Record<string, any>>, maxAge = config.cache.contentExpire, refresh = true) => {
|
||||
if (typeof key !== 'string') {
|
||||
throw new TypeError('Cache key must be a string');
|
||||
}
|
||||
let v = await cacheModule.get(key, refresh);
|
||||
if (v) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = JSON.parse(v);
|
||||
} catch {
|
||||
parsed = null;
|
||||
}
|
||||
if (parsed) {
|
||||
v = parsed;
|
||||
}
|
||||
|
||||
return v;
|
||||
} else {
|
||||
const value = await getValueFunc();
|
||||
cacheModule.set(key, value, maxAge);
|
||||
|
||||
return value;
|
||||
}
|
||||
},
|
||||
globalCache,
|
||||
};
|
||||
@@ -1,5 +1,6 @@
|
||||
const { LRUCache } = require('lru-cache');
|
||||
const config = require('@/config').value;
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { config } from '@/config';
|
||||
import type CacheModule from './base';
|
||||
|
||||
const status = { available: false };
|
||||
|
||||
@@ -10,10 +11,10 @@ const memoryCache = new LRUCache({
|
||||
|
||||
status.available = true;
|
||||
|
||||
module.exports = {
|
||||
get: (key, refresh = true) => {
|
||||
export default {
|
||||
get: (key: string, refresh = true) => {
|
||||
if (key && status.available) {
|
||||
let value = memoryCache.get(key, { updateAgeOnGet: refresh });
|
||||
let value = memoryCache.get(key, { updateAgeOnGet: refresh }) as string | undefined;
|
||||
if (value) {
|
||||
value = value + '';
|
||||
}
|
||||
@@ -33,4 +34,4 @@ module.exports = {
|
||||
},
|
||||
clients: { memoryCache },
|
||||
status,
|
||||
};
|
||||
} as CacheModule;
|
||||
+10
-9
@@ -1,6 +1,7 @@
|
||||
const config = require('@/config').value;
|
||||
const Redis = require('ioredis');
|
||||
const logger = require('@/utils/logger');
|
||||
import { config } from '@/config';
|
||||
import Redis from 'ioredis';
|
||||
import logger from '@/utils/logger';
|
||||
import type CacheModule from './base';
|
||||
|
||||
const redisClient = new Redis(config.redis.url);
|
||||
|
||||
@@ -18,15 +19,15 @@ redisClient.on('connect', () => {
|
||||
logger.info('Redis connected.');
|
||||
});
|
||||
|
||||
const getCacheTtlKey = (key) => {
|
||||
const getCacheTtlKey = (key: string) => {
|
||||
if (key.startsWith('rsshub:cacheTtl:')) {
|
||||
throw new Error('"rsshub:cacheTtl:" prefix is reserved for the internal usage, please change your cache key'); // blocking any attempt to get/set the cacheTtl
|
||||
}
|
||||
return `rsshub:cacheTtl:${key}`;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
get: async (key, refresh = true) => {
|
||||
export default {
|
||||
get: async (key: string, refresh = true) => {
|
||||
if (key && status.available) {
|
||||
const cacheTtlKey = getCacheTtlKey(key);
|
||||
let [value, cacheTtl] = await redisClient.mget(key, cacheTtlKey);
|
||||
@@ -35,7 +36,7 @@ module.exports = {
|
||||
redisClient.expire(cacheTtlKey, cacheTtl);
|
||||
} else {
|
||||
// if cacheTtl is not set, that means the cache expire time is contentExpire
|
||||
cacheTtl = config.cache.contentExpire;
|
||||
cacheTtl = config.cache.contentExpire + '';
|
||||
// dont save cacheTtl to Redis, as it is the default value
|
||||
// redisClient.set(cacheTtlKey, cacheTtl, 'EX', cacheTtl);
|
||||
}
|
||||
@@ -45,7 +46,7 @@ module.exports = {
|
||||
return value;
|
||||
}
|
||||
},
|
||||
set: (key, value, maxAge = config.cache.contentExpire) => {
|
||||
set: (key: string, value?: string | Record<string, any>, maxAge = config.cache.contentExpire) => {
|
||||
if (!status.available) {
|
||||
return;
|
||||
}
|
||||
@@ -65,4 +66,4 @@ module.exports = {
|
||||
},
|
||||
clients: { redisClient },
|
||||
status,
|
||||
};
|
||||
} as CacheModule;
|
||||
@@ -1,32 +1,32 @@
|
||||
const { parseDate } = require('@/utils/parse-date');
|
||||
import parseDate from '@/utils/parse-date';
|
||||
const title = require('title');
|
||||
|
||||
// convert a string into title case
|
||||
const toTitleCase = (str) => title(str);
|
||||
const toTitleCase = (str: string) => title(str);
|
||||
|
||||
const rWhiteSpace = /\s+/;
|
||||
const rAllWhiteSpace = /\s+/g;
|
||||
|
||||
// collapse all whitespaces into a single space (like "white-space: normal;" would do), and trim
|
||||
const collapseWhitespace = (str) => {
|
||||
const collapseWhitespace = (str: string) => {
|
||||
if (str && rWhiteSpace.test(str)) {
|
||||
return str.replaceAll(rAllWhiteSpace, ' ').trim();
|
||||
}
|
||||
return str;
|
||||
};
|
||||
|
||||
const convertDateToISO8601 = (date) => {
|
||||
const convertDateToISO8601 = (date: string | Date) => {
|
||||
if (!date) {
|
||||
return date;
|
||||
}
|
||||
if (typeof date !== 'object') {
|
||||
// some routes may call `.toUTCString()` before passing the date to ctx...
|
||||
date = parseDate(date);
|
||||
date = parseDate.parse(date);
|
||||
}
|
||||
return date.toISOString();
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
export default {
|
||||
toTitleCase,
|
||||
collapseWhitespace,
|
||||
convertDateToISO8601,
|
||||
@@ -0,0 +1,41 @@
|
||||
import { promises as fs } from 'node:fs';
|
||||
|
||||
import { ImportedModulesPrivateOptions } from './types.d';
|
||||
|
||||
/**
|
||||
* Async version of the directory reader.
|
||||
*
|
||||
* I use *for loop* instead of forEach because it is faster.
|
||||
* I define the *counter* variable outside the loop because it is faster. Because it is not redefined every iteration.
|
||||
* @param {ImportedModulesPrivateOptions} options - The private options generated by the preparePrivateOptions function.
|
||||
* @param {string} targetDirectoryPath - The path to the directory to import modules from. Required for recursive calls.
|
||||
* @returns {Promise<string[]>} An object containing all imported modules.
|
||||
*/
|
||||
export default async function readDirectoryAsync(
|
||||
options: ImportedModulesPrivateOptions,
|
||||
targetDirectoryPath: string,
|
||||
): Promise<string[]> {
|
||||
const receivedItemsPaths = await fs.readdir(targetDirectoryPath);
|
||||
|
||||
const result = await Promise.all(
|
||||
receivedItemsPaths.map(async (itemPath) => {
|
||||
const receivedFilesPaths = [];
|
||||
|
||||
const relativeItemPath = `${targetDirectoryPath}/${itemPath}`;
|
||||
|
||||
const stat = await fs.stat(relativeItemPath);
|
||||
|
||||
if (stat.isDirectory() && options.includeSubdirectories) {
|
||||
const files = await readDirectoryAsync(options, relativeItemPath);
|
||||
|
||||
receivedFilesPaths.push(...files);
|
||||
} else if (stat.isFile()) {
|
||||
receivedFilesPaths.push('/' + relativeItemPath);
|
||||
}
|
||||
|
||||
return receivedFilesPaths;
|
||||
}),
|
||||
);
|
||||
|
||||
return result.flat();
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import * as fs from 'node:fs';
|
||||
|
||||
import { ImportedModulesPrivateOptions } from './types.d';
|
||||
|
||||
/**
|
||||
* Sync version of the directory reader.
|
||||
*
|
||||
* !important this function will block the event loop until it finishes.
|
||||
* I use *for loop* instead of forEach because it is faster.
|
||||
* I define the *counter* variable outside the loop because it is faster. Because it is not redefined every iteration.
|
||||
* @param {ImportedModulesPrivateOptions} options - The private options generated by the preparePrivateOptions function.
|
||||
* @param {string} targetDirectoryPath - The path to the directory to import modules from. Required for recursive calls.
|
||||
* @returns {string[]} An array of file paths.
|
||||
*/
|
||||
export default function readDirectorySync(
|
||||
options: ImportedModulesPrivateOptions,
|
||||
targetDirectoryPath: string,
|
||||
): string[] {
|
||||
const receivedItemsPaths = fs.readdirSync(targetDirectoryPath);
|
||||
const receivedDirectoriesPaths = [];
|
||||
const receivedFilesPaths = [];
|
||||
|
||||
// Get the files and directories paths from the target directory.
|
||||
let itemsCounter = 0;
|
||||
for (itemsCounter; itemsCounter < receivedItemsPaths.length; itemsCounter += 1) {
|
||||
const itemPath = `${targetDirectoryPath}/${receivedItemsPaths[itemsCounter]}`;
|
||||
const stat = fs.statSync(itemPath);
|
||||
|
||||
if (stat.isDirectory() && options.includeSubdirectories) {
|
||||
receivedDirectoriesPaths.push(itemPath);
|
||||
} else if (stat.isFile()) {
|
||||
receivedFilesPaths.push('/' + itemPath);
|
||||
}
|
||||
}
|
||||
|
||||
// Recursively get the files paths from the subdirectories.
|
||||
let directoriesCounter = 0;
|
||||
for (directoriesCounter; directoriesCounter < receivedDirectoriesPaths.length; directoriesCounter += 1) {
|
||||
const files = readDirectorySync(options, receivedDirectoriesPaths[directoriesCounter] as string);
|
||||
|
||||
receivedFilesPaths.push(...files);
|
||||
}
|
||||
|
||||
// Return the files paths.
|
||||
return receivedFilesPaths;
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
import * as path from 'node:path';
|
||||
|
||||
import readDirectoryAsync from './directory-reader-async';
|
||||
import readDirectorySync from './directory-reader-sync';
|
||||
import { ImportedModules, ImportedModulesPrivateOptions } from './types.d';
|
||||
|
||||
const VALID_IMPORT_EXTENSIONS = new Set(['.js', '.mjs', '.ts', '.json']);
|
||||
|
||||
const handlers = { async: asyncHandler, sync: syncHandler };
|
||||
|
||||
/**
|
||||
* Synchronously import modules from the specified directory.
|
||||
* @param {ImportedModulesPrivateOptions} options - The private options generated by the preparePrivateOptions function.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function syncHandler(options: ImportedModulesPrivateOptions): ImportedModules {
|
||||
const modules = {};
|
||||
|
||||
const filesPaths = readDirectorySync(options, options.targetDirectoryPath);
|
||||
|
||||
let index = 0;
|
||||
|
||||
for (const filePath of filesPaths) {
|
||||
const isModuleImported = importModule(filePath, index, options, modules);
|
||||
|
||||
if (isModuleImported) index += 1;
|
||||
if (index === options.limit) break;
|
||||
}
|
||||
|
||||
return modules;
|
||||
}
|
||||
|
||||
/**
|
||||
* Asynchronously import modules from the specified directory.
|
||||
* @param {ImportedModulesPrivateOptions} options - The private options generated by the preparePrivateOptions function.
|
||||
* @returns {Promise<ImportedModules>} An object containing all imported modules.
|
||||
*/
|
||||
async function asyncHandler(options: ImportedModulesPrivateOptions): Promise<ImportedModules> {
|
||||
const modules = {};
|
||||
|
||||
const filesPaths = await readDirectoryAsync(options, options.targetDirectoryPath);
|
||||
|
||||
let index = 0;
|
||||
|
||||
for (const filePath of filesPaths) {
|
||||
const isModuleImported = importModule(filePath, index, options, modules);
|
||||
|
||||
if (isModuleImported) index += 1;
|
||||
if (index === options.limit) break;
|
||||
}
|
||||
|
||||
return modules;
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a module and add it to the modules object.
|
||||
* @param {string} filePath - The path to the file to import.
|
||||
* @param {number} index - The index of the module.
|
||||
* @param {ImportedModulesPrivateOptions} options - The private options generated by the preparePrivateOptions function.
|
||||
* @param {ImportedModules} modules - The object containing all imported modules.
|
||||
* @returns {boolean} Whether the module was imported or not.
|
||||
*/
|
||||
function importModule(
|
||||
filePath: string,
|
||||
index: number,
|
||||
options: ImportedModulesPrivateOptions,
|
||||
modules: ImportedModules,
|
||||
) {
|
||||
const { name: fileName, ext: fileExtension } = path.parse(filePath);
|
||||
const isValidModuleExtension = VALID_IMPORT_EXTENSIONS.has(fileExtension);
|
||||
const isDeclarationFile = filePath.endsWith('.d.ts');
|
||||
const isValidFilePath = options.importPattern ? options.importPattern.test(filePath) : true;
|
||||
|
||||
if (!isValidModuleExtension) return false;
|
||||
if (!isValidFilePath) return false;
|
||||
if (isDeclarationFile) return false;
|
||||
|
||||
const relativeModulePath = filePath.slice(options.targetDirectoryPath.length + 1);
|
||||
|
||||
// eslint-disable-next-line security/detect-non-literal-require, @typescript-eslint/no-var-requires, unicorn/prefer-module
|
||||
const importedModule = require(filePath) as unknown;
|
||||
|
||||
modules[relativeModulePath] = importedModule;
|
||||
|
||||
if (options.callback) {
|
||||
options.callback(fileName, relativeModulePath, importedModule, index);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Import all modules from the specified directory synchronously or asynchronously.
|
||||
* @param {ImportedModulesPrivateOptions} options - The private options generated by the preparePrivateOptions function.
|
||||
* @returns {ImportedModules | Promise<ImportedModules>} An object containing all imported modules.
|
||||
*/
|
||||
export default function importModules(
|
||||
options: ImportedModulesPrivateOptions,
|
||||
): ImportedModules | Promise<ImportedModules> {
|
||||
if (!handlers[options.importMode]) {
|
||||
throw new Error(`Expected sync or async import method, but got: ${options.importMode}`);
|
||||
}
|
||||
|
||||
return handlers[options.importMode](options);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import importModules from './import-modules';
|
||||
import preparePrivateOptions from './prepare-private-options';
|
||||
import {
|
||||
ImportedModules,
|
||||
ImportedModulesPublicOptions,
|
||||
ImportModulesCallback,
|
||||
ImportModulesInputArguments,
|
||||
ImportModulesMode,
|
||||
} from './types.d';
|
||||
|
||||
/**
|
||||
* Import modules from the current directory synchronously
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import modules from the current directory synchronously and call the provided callback for each imported module.
|
||||
* @param {ImportModulesCallback} callback - The callback function to call for each imported module.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(callback: ImportModulesCallback): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import modules from the specified directory synchronously.
|
||||
* @param {string} targetDirectoryPath - The path to the directory to import modules from.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(targetDirectoryPath: string): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import all modules from the specified directory synchronously and call the provided callback for each imported module.
|
||||
* @param {string} targetDirectoryPath - The path to the directory to import modules from.
|
||||
* @param {ImportModulesCallback} callback - The callback function to call for each imported module.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(targetDirectoryPath: string, callback: ImportModulesCallback): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import all modules from the specified directory synchronously or asynchronously.
|
||||
* @param {string} targetDirectoryPath - The path to the directory to import modules from.
|
||||
* @param {ImportModulesMode} mode - The import mode. Can be 'sync' or 'async'.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(targetDirectoryPath: string, mode: ImportModulesMode): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import all modules from the specified directory synchronously or asynchronously and call the provided callback for each imported module.
|
||||
* @param {string} targetDirectoryPath - The path to the directory to import modules from.
|
||||
* @param {ImportModulesMode} importMode - The import mode. Can be 'sync' or 'async'.
|
||||
* @param {ImportModulesCallback} callback - The callback function to call for each imported module.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(
|
||||
targetDirectoryPath: string,
|
||||
importMode: ImportModulesMode,
|
||||
callback: ImportModulesCallback,
|
||||
): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import all modules from the specified directory
|
||||
* @param {ImportedModulesPublicOptions} options - The options object.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(options: ImportedModulesPublicOptions): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import all modules from the specified directory and call the provided callback for each imported module.
|
||||
* @param {ImportedModulesPublicOptions} options - The options object.
|
||||
* @param {ImportModulesCallback} callback - The callback function to call for each imported module.
|
||||
* @returns {ImportedModules} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(options: ImportedModulesPublicOptions, callback: ImportModulesCallback): ImportedModules;
|
||||
|
||||
/**
|
||||
* Import all modules from the specified directory with the given options and call the provided callback for each imported module.
|
||||
* @param {ImportModulesInputArguments} arguments_ - The arguments.
|
||||
* @returns {ImportedModules|Promise<ImportedModules>} An object containing all imported modules.
|
||||
*/
|
||||
function directoryImport(...arguments_: ImportModulesInputArguments): ImportedModules | Promise<ImportedModules> {
|
||||
const options = preparePrivateOptions(...arguments_);
|
||||
|
||||
try {
|
||||
return importModules(options);
|
||||
} catch (error) {
|
||||
Object.assign(error as Error, options);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export { directoryImport };
|
||||
@@ -0,0 +1,121 @@
|
||||
import * as path from 'node:path';
|
||||
|
||||
import {
|
||||
ImportedModulesPrivateOptions,
|
||||
ImportedModulesPublicOptions,
|
||||
ImportModulesInputArguments,
|
||||
ImportModulesMode,
|
||||
} from './types.d';
|
||||
|
||||
const getDefaultOptions = (): ImportedModulesPrivateOptions => {
|
||||
const options = {
|
||||
includeSubdirectories: true,
|
||||
importMode: 'sync' as ImportModulesMode,
|
||||
importPattern: /.*/,
|
||||
limit: Number.POSITIVE_INFINITY,
|
||||
callerFilePath: '/',
|
||||
callerDirectoryPath: '/',
|
||||
targetDirectoryPath: '/',
|
||||
};
|
||||
|
||||
options.callerFilePath = (new Error('functional-error').stack as string)
|
||||
.split('\n')[4]
|
||||
?.match(/at file:\/\/(.+):\d+|at .+ \((.+):\d+:\d+\)/)?.slice(1).find(Boolean) as string;
|
||||
|
||||
options.callerDirectoryPath = options.callerFilePath?.split('/').slice(0, -1).join('/');
|
||||
options.targetDirectoryPath = options.callerDirectoryPath;
|
||||
|
||||
return options;
|
||||
};
|
||||
|
||||
/**
|
||||
* Prepare the options object from the provided arguments.
|
||||
* @param {...any} arguments_ - The arguments.
|
||||
* @returns {ImportedModulesPrivateOptions} The options object.
|
||||
*/
|
||||
export default function preparePrivateOptions(
|
||||
...arguments_: ImportModulesInputArguments
|
||||
): ImportedModulesPrivateOptions {
|
||||
const options = { ...getDefaultOptions() };
|
||||
|
||||
// * Check first argument
|
||||
|
||||
// ** If user provided nothing as first argument,
|
||||
// ** it means that he wants to use the default options
|
||||
// ** return the default options
|
||||
if (arguments_[0] === undefined) {
|
||||
return options;
|
||||
}
|
||||
|
||||
// ** If user provided an object as first argument,
|
||||
// ** it means that he wants to set custom options
|
||||
// ** use it to oweverwrite the default options
|
||||
else if (typeof arguments_[0] === 'object') {
|
||||
const result = {
|
||||
...getDefaultOptions(),
|
||||
...(arguments_[0] as ImportedModulesPublicOptions),
|
||||
};
|
||||
|
||||
result.targetDirectoryPath = path.resolve(result.callerDirectoryPath, result.targetDirectoryPath);
|
||||
result.callback = typeof arguments_[1] === 'function' ? arguments_[1] : undefined;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ** If user provided a string
|
||||
// ** it means that he wants to set the target directory path as first argument
|
||||
// ** use it to oweverwrite the default options
|
||||
else if (typeof arguments_[0] === 'string') {
|
||||
options.targetDirectoryPath = path.resolve(options.callerDirectoryPath, arguments_[0]);
|
||||
}
|
||||
|
||||
// ** If user provided a function
|
||||
// ** it means that he wants to set the callback as first argument
|
||||
// ** use it to oweverwrite the default options
|
||||
else if (typeof arguments_[0] === 'function') {
|
||||
options.callback = arguments_[0];
|
||||
}
|
||||
|
||||
// ** If user provided something else
|
||||
// ** it means that he made a mistake
|
||||
// ** throw an error
|
||||
else {
|
||||
throw new TypeError(
|
||||
`Expected undefined, object, string or function as first argument, but got: ${typeof arguments_[0]}`,
|
||||
);
|
||||
}
|
||||
|
||||
// * Check second argument
|
||||
|
||||
// ** If user provided string as second argument
|
||||
// ** it means that he wants to set the import mode (sync or async) as second argument
|
||||
// ** use it to oweverwrite the default options
|
||||
if (typeof arguments_[1] === 'string') {
|
||||
// ** But if the import mode is not sync or async
|
||||
// ** throw an error
|
||||
if (arguments_[1] !== 'sync' && arguments_[1] !== 'async') {
|
||||
throw new TypeError(`Expected sync or async as second argument, but got: ${arguments_[1]}`);
|
||||
}
|
||||
|
||||
options.importMode = arguments_[1];
|
||||
}
|
||||
|
||||
// ** If user provided a function as second argument
|
||||
// ** it means that he wants to set the callback as second argument
|
||||
// ** use it to oweverwrite the default options
|
||||
else if (typeof arguments_[1] === 'function') {
|
||||
options.callback = arguments_[1];
|
||||
}
|
||||
|
||||
// * Check third argument
|
||||
|
||||
// ** If user provided a function as third argument
|
||||
// ** it means that he wants to set the callback as third argument
|
||||
// ** use it to oweverwrite the default options
|
||||
if (typeof arguments_[2] === 'function') {
|
||||
options.callback = arguments_[2];
|
||||
}
|
||||
|
||||
// * Return the options object
|
||||
return options;
|
||||
}
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
export type ModuleName = string;
|
||||
export type ModulePath = string;
|
||||
export type ModuleData = unknown;
|
||||
export type ModuleIndex = number;
|
||||
|
||||
export type ImportModulesMode = 'sync' | 'async';
|
||||
export type ImportModulesCallback = (name: ModuleName, path: ModulePath, data: ModuleData, index: ModuleIndex) => void;
|
||||
|
||||
export type ImportModulesInputArguments = [
|
||||
targetDirectoryPathOrOptionsOrCallback?: string | ImportedModulesPublicOptions | ImportModulesCallback,
|
||||
modeOrCallback?: ImportModulesMode | ImportModulesCallback,
|
||||
callback?: ImportModulesCallback,
|
||||
];
|
||||
|
||||
export interface ImportedModules {
|
||||
[key: ModulePath]: ModuleData;
|
||||
}
|
||||
|
||||
export interface ImportedModulesPublicOptions {
|
||||
includeSubdirectories?: boolean;
|
||||
callerDirectoryPath?: string;
|
||||
targetDirectoryPath?: string;
|
||||
importPattern?: RegExp;
|
||||
importMode?: ImportModulesMode;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface ImportedModulesPrivateOptions extends Required<ImportedModulesPublicOptions> {
|
||||
callback?: ImportModulesCallback;
|
||||
|
||||
callerFilePath: string;
|
||||
callerDirectoryPath: string;
|
||||
}
|
||||
@@ -1,15 +1,19 @@
|
||||
const logger = require('./logger');
|
||||
const config = require('@/config').value;
|
||||
const got = require('got');
|
||||
import logger from "@/utils/logger";
|
||||
import { config } from "@/config";
|
||||
import got, { type Response, type NormalizedOptions, type Options } from "got";
|
||||
|
||||
const custom = got.extend({
|
||||
const custom: typeof got & {
|
||||
all?: <T>(list: Array<Promise<T>>) => Promise<Array<T>>;
|
||||
} = got.extend({
|
||||
retry: {
|
||||
limit: config.requestRetry,
|
||||
statusCodes: [400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 421, 422, 423, 424, 426, 428, 429, 431, 451, 500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511, 521, 522, 524],
|
||||
},
|
||||
hooks: {
|
||||
beforeRetry: [
|
||||
(options, err, count) => {
|
||||
(options: NormalizedOptions & {
|
||||
retryCount?: number;
|
||||
}, err, count) => {
|
||||
logger.error(`Request ${options.url} fail, retry attempt #${count}: ${err}`);
|
||||
options.retryCount = count;
|
||||
},
|
||||
@@ -20,18 +24,23 @@ const custom = got.extend({
|
||||
},
|
||||
],
|
||||
afterResponse: [
|
||||
(response) => {
|
||||
(response: Response & {
|
||||
data?: Record<string, any> | string;
|
||||
status?: number;
|
||||
}) => {
|
||||
try {
|
||||
response.data = JSON.parse(response.body);
|
||||
response.data = JSON.parse(response.body as string);
|
||||
} catch {
|
||||
response.data = response.body;
|
||||
response.data = response.body as string;
|
||||
}
|
||||
response.status = response.statusCode;
|
||||
return response;
|
||||
},
|
||||
],
|
||||
init: [
|
||||
(options) => {
|
||||
(options: Options & {
|
||||
data?: string;
|
||||
}) => {
|
||||
// compatible with axios api
|
||||
if (options && options.data) {
|
||||
options.body = options.body || options.data;
|
||||
@@ -46,4 +55,4 @@ const custom = got.extend({
|
||||
});
|
||||
custom.all = (list) => Promise.all(list);
|
||||
|
||||
module.exports = custom;
|
||||
export default custom;
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Context } from "hono";
|
||||
import { ReadableStream } from "node:stream/web";
|
||||
|
||||
export const getRouteNameFromPath = (path: string) => {
|
||||
const p = path.split('/').filter(Boolean);
|
||||
if (p.length > 0) {
|
||||
return p[0];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export const getIp = (ctx: Context) => {
|
||||
return ctx.req.header('X-Real-IP') || ctx.req.header('X-Forwarded-For')
|
||||
}
|
||||
|
||||
export const streamToString = async (stream: ReadableStream) => {
|
||||
const reader = stream.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
const result = new TextDecoder('utf-8').decode(
|
||||
new Uint8Array(chunks.reduce((a, c) => a + c.length, 0))
|
||||
);
|
||||
return result;
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
const path = require('path');
|
||||
const winston = require('winston');
|
||||
const config = require('@/config').value;
|
||||
import * as path from 'node:path';
|
||||
import winston from 'winston';
|
||||
import { config } from '@/config';
|
||||
|
||||
let transports = [];
|
||||
let transports: (typeof winston.transports.File)[] = [];
|
||||
if (!config.noLogfiles) {
|
||||
transports = [
|
||||
new winston.transports.File({
|
||||
@@ -43,4 +43,4 @@ if (!config.isPackage) {
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = logger;
|
||||
export default logger;
|
||||
@@ -1,5 +0,0 @@
|
||||
const crypto = require('crypto');
|
||||
|
||||
module.exports = function md5(date) {
|
||||
return crypto.createHash('md5').update(date).digest('hex');
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
import crypto from 'crypto';
|
||||
|
||||
export default function md5(date: string) {
|
||||
return crypto.createHash('md5').update(date).digest('hex');
|
||||
};
|
||||
@@ -1,8 +1,13 @@
|
||||
const dayjs = require('dayjs');
|
||||
dayjs.extend(require('dayjs/plugin/customParseFormat'));
|
||||
dayjs.extend(require('dayjs/plugin/duration'));
|
||||
dayjs.extend(require('dayjs/plugin/isSameOrBefore'));
|
||||
dayjs.extend(require('dayjs/plugin/weekday'));
|
||||
import dayjs from 'dayjs';
|
||||
import customParseFormat from 'dayjs/plugin/customParseFormat';
|
||||
import duration from 'dayjs/plugin/duration';
|
||||
import isSameOrBefore from 'dayjs/plugin/isSameOrBefore';
|
||||
import weekday from 'dayjs/plugin/weekday';
|
||||
|
||||
dayjs.extend(customParseFormat);
|
||||
dayjs.extend(duration);
|
||||
dayjs.extend(isSameOrBefore);
|
||||
dayjs.extend(weekday);
|
||||
|
||||
const words = [
|
||||
{
|
||||
@@ -92,7 +97,7 @@ const patternSize = Object.keys(patterns).length;
|
||||
* 预处理日期字符串
|
||||
* @param {String} date 原始日期字符串
|
||||
*/
|
||||
const toDate = (date) =>
|
||||
const toDate = (date: string) =>
|
||||
date
|
||||
.toLowerCase()
|
||||
.replaceAll(/(^an?\s)|(\san?\s)/g, '1') // 替换 `a` 和 `an` 为 `1`
|
||||
@@ -104,8 +109,8 @@ const toDate = (date) =>
|
||||
* 用于描述时间长度
|
||||
* @param {Array.<String>} matches 所有匹配结果
|
||||
*/
|
||||
const toDurations = (matches) => {
|
||||
const durations = {};
|
||||
const toDurations = (matches: string[]) => {
|
||||
const durations: Record<string, string> = {};
|
||||
|
||||
let p = 0;
|
||||
for (const m of matches) {
|
||||
@@ -120,9 +125,9 @@ const toDurations = (matches) => {
|
||||
return durations;
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
parseDate: (date, ...options) => dayjs(date, ...options).toDate(),
|
||||
parseRelativeDate: (date) => {
|
||||
export default {
|
||||
parse: (date: string, ...options: any) => dayjs(date, ...options).toDate(),
|
||||
parseRelativeDate: (date: string) => {
|
||||
// 预处理日期字符串 date
|
||||
|
||||
const theDate = toDate(date);
|
||||
@@ -136,49 +141,53 @@ module.exports = {
|
||||
|
||||
const lastMatch = matches.pop();
|
||||
|
||||
// 若最后的时间单元含有 `前`、`以前`、`之前` 等标识字段，减去相应的时间长度
|
||||
// 如 `1分10秒前`
|
||||
if (lastMatch) {
|
||||
// 若最后的时间单元含有 `前`、`以前`、`之前` 等标识字段，减去相应的时间长度
|
||||
// 如 `1分10秒前`
|
||||
|
||||
const beforeMatches = /(.*)(?:[之以]?前|ago)$/.exec(lastMatch);
|
||||
if (beforeMatches) {
|
||||
matches.push(beforeMatches[1]);
|
||||
return dayjs()
|
||||
.subtract(dayjs.duration(toDurations(matches)))
|
||||
.toDate();
|
||||
}
|
||||
const beforeMatches = /(.*)(?:[之以]?前|ago)$/.exec(lastMatch);
|
||||
if (beforeMatches) {
|
||||
matches.push(beforeMatches[1]);
|
||||
return dayjs()
|
||||
.subtract(dayjs.duration(toDurations(matches)))
|
||||
.toDate();
|
||||
}
|
||||
|
||||
// 若最后的时间单元含有 `后`、`以后`、`之后` 等标识字段，加上相应的时间长度
|
||||
// 如 `1分10秒后`
|
||||
// 若最后的时间单元含有 `后`、`以后`、`之后` 等标识字段，加上相应的时间长度
|
||||
// 如 `1分10秒后`
|
||||
|
||||
const afterMatches = /(?:^in(.*)|(.*)[之以]?[后後])$/.exec(lastMatch);
|
||||
if (afterMatches) {
|
||||
matches.push(afterMatches[1] ?? afterMatches[2]);
|
||||
return dayjs()
|
||||
.add(dayjs.duration(toDurations(matches)))
|
||||
.toDate();
|
||||
}
|
||||
|
||||
// 以下处理日期字符串 date 含有特殊词的情形
|
||||
// 如 `今天1点10分`
|
||||
|
||||
matches.push(lastMatch);
|
||||
const firstMatch = matches.shift();
|
||||
|
||||
for (const w of words) {
|
||||
const wordMatches = w.regExp.exec(firstMatch);
|
||||
if (wordMatches) {
|
||||
matches.unshift(wordMatches[1]);
|
||||
|
||||
// 取特殊词对应日零时为起点，加上相应的时间长度
|
||||
|
||||
return w.startAt
|
||||
.set('hour', 0)
|
||||
.set('minute', 0)
|
||||
.set('second', 0)
|
||||
.set('millisecond', 0)
|
||||
const afterMatches = /(?:^in(.*)|(.*)[之以]?[后後])$/.exec(lastMatch);
|
||||
if (afterMatches) {
|
||||
matches.push(afterMatches[1] ?? afterMatches[2]);
|
||||
return dayjs()
|
||||
.add(dayjs.duration(toDurations(matches)))
|
||||
.toDate();
|
||||
}
|
||||
|
||||
// 以下处理日期字符串 date 含有特殊词的情形
|
||||
// 如 `今天1点10分`
|
||||
|
||||
matches.push(lastMatch);
|
||||
}
|
||||
const firstMatch = matches.shift();
|
||||
|
||||
if (firstMatch) {
|
||||
for (const w of words) {
|
||||
const wordMatches = w.regExp.exec(firstMatch);
|
||||
if (wordMatches) {
|
||||
matches.unshift(wordMatches[1]);
|
||||
|
||||
// 取特殊词对应日零时为起点，加上相应的时间长度
|
||||
|
||||
return w.startAt
|
||||
.set('hour', 0)
|
||||
.set('minute', 0)
|
||||
.set('second', 0)
|
||||
.set('millisecond', 0)
|
||||
.add(dayjs.duration(toDurations(matches)))
|
||||
.toDate();
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 若日期字符串 date 不匹配 patterns 中所有模式，则默认为 `特殊词 + 标准时间格式` 的情形，此时直接将特殊词替换为对应日期
|
||||
@@ -1,4 +1,4 @@
|
||||
const { randUserAgent } = require('@tonyrl/rand-user-agent');
|
||||
import { randUserAgent } from '@tonyrl/rand-user-agent';
|
||||
require.resolve('@tonyrl/rand-user-agent/data/user-agents.json'); // workaround for @vercel/nft removing datastore
|
||||
|
||||
/**
|
||||
@@ -10,7 +10,7 @@ require.resolve('@tonyrl/rand-user-agent/data/user-agents.json'); // workaround
|
||||
* @param {string} randUserAgent.device Name of a device, case-insensitive. `desktop`, `mobile` or `tablet`.
|
||||
* @returns A random useragent for the given specifications.
|
||||
*/
|
||||
module.exports = ({ browser = 'chrome', os = 'mac os', device = 'desktop' }) => {
|
||||
export default ({ browser = 'chrome', os = 'mac os', device = 'desktop' }) => {
|
||||
device = device.toLowerCase();
|
||||
browser = browser.toLowerCase();
|
||||
os = os.toLowerCase();
|
||||
@@ -1,11 +0,0 @@
|
||||
const art = require('art-template');
|
||||
const json = require('@/views/json');
|
||||
const rss3Ums = require('@/views/rss3-ums');
|
||||
|
||||
// We may add more control over it later
|
||||
|
||||
module.exports = {
|
||||
art,
|
||||
json, // This should be used by RSSHub middleware only.
|
||||
rss3Ums,
|
||||
};
|
||||
@@ -0,0 +1,11 @@
|
||||
import art from 'art-template';
|
||||
import json from '@/views/json';
|
||||
import rss3Ums from '@/views/rss3-ums';
|
||||
|
||||
// We may add more control over it later
|
||||
|
||||
export default {
|
||||
art,
|
||||
json, // This should be used by RSSHub middleware only.
|
||||
rss3Ums,
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
module.exports = {
|
||||
'/topic/text/:id': ['HenryQW'],
|
||||
'/topic/:id/:showUid?': ['DIYgod', 'prnake'],
|
||||
'/user/:id': ['DIYgod', 'prnake'],
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
module.exports = {
|
||||
'okjike.com': {
|
||||
_name: '即刻',
|
||||
m: [
|
||||
{
|
||||
title: '用户动态',
|
||||
docs: 'https://docs.rsshub.app/routes/social-media#ji-ke',
|
||||
source: '/users/:uid',
|
||||
target: '/jike/user/:uid',
|
||||
},
|
||||
{
|
||||
title: '用户动态',
|
||||
docs: 'https://docs.rsshub.app/routes/social-media#ji-ke',
|
||||
source: '/reposts/:repostId',
|
||||
target: (params, url, document) => {
|
||||
const uid = document.querySelector('.avatar').getAttribute('href').replace('/users/', '');
|
||||
return uid ? `/jike/user/${uid}` : '';
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '圈子',
|
||||
docs: 'https://docs.rsshub.app/routes/social-media#ji-ke',
|
||||
source: '/topics/:id',
|
||||
target: '/jike/topic/:id',
|
||||
},
|
||||
{
|
||||
title: '圈子 - 纯文字',
|
||||
docs: 'https://docs.rsshub.app/routes/social-media#ji-ke',
|
||||
source: '/topics/:id',
|
||||
target: '/jike/topic/text/:id',
|
||||
},
|
||||
],
|
||||
web: [
|
||||
{
|
||||
title: '用户动态',
|
||||
docs: 'https://docs.rsshub.app/routes/social-media#ji-ke',
|
||||
source: '/u/:uid',
|
||||
target: '/jike/user/:uid',
|
||||
},
|
||||
{
|
||||
title: '圈子',
|
||||
docs: 'https://docs.rsshub.app/routes/social-media#ji-ke',
|
||||
source: '/topic/:id',
|
||||
target: '/jike/topic/:id',
|
||||
},
|
||||
{
|
||||
title: '圈子 - 纯文字',
|
||||
docs: 'https://docs.rsshub.app/routes/social-media#ji-ke',
|
||||
source: '/topic/:id',
|
||||
target: '/jike/topic/text/:id',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,5 @@
|
||||
module.exports = (router) => {
|
||||
router.get('/topic/text/:id', require('./topic-text'));
|
||||
router.get('/topic/:id/:showUid?', require('./topic'));
|
||||
router.get('/user/:id', require('./user'));
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
const dayjs = require('dayjs');
|
||||
const { constructTopicEntry } = require('./utils');
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const id = ctx.params.id;
|
||||
const topicUrl = `https://m.okjike.com/topics/${id}`;
|
||||
|
||||
const data = await constructTopicEntry(ctx, topicUrl);
|
||||
|
||||
if (data) {
|
||||
ctx.state.data.item = data.posts.map((item) => {
|
||||
const date = dayjs(item.createdAt);
|
||||
return {
|
||||
title: `${data.topic.content} ${date.format('MM月DD日')}`,
|
||||
description: item.content.replaceAll('\n', '<br>'),
|
||||
pubDate: date.toDate(),
|
||||
link: `https://m.okjike.com/originalPosts/${item.id}`,
|
||||
};
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,48 @@
|
||||
const got = require('@/utils/got');
|
||||
const { topicDataHanding } = require('./utils');
|
||||
const cheerio = require('cheerio');
|
||||
const dayjs = require('dayjs');
|
||||
const { constructTopicEntry } = require('@/v3/jike/utils');
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const id = ctx.req.param('id');
|
||||
const topicUrl = `https://m.okjike.com/topics/${id}`;
|
||||
|
||||
const data = await constructTopicEntry(ctx, topicUrl);
|
||||
|
||||
if (data) {
|
||||
let item = topicDataHanding(data, ctx);
|
||||
if (id === '553870e8e4b0cafb0a1bef68' || id === '55963702e4b0d84d2c30ce6f') {
|
||||
item = await Promise.all(
|
||||
item.map(async (one) => {
|
||||
const item = { ...one };
|
||||
const regResult = /https:\/\/www\.okjike\.com\/medium\/[\dA-Za-z]*/.exec(item.description);
|
||||
if (regResult) {
|
||||
const newsUrl = regResult[0];
|
||||
item.description = await ctx.cache.tryGet(newsUrl, async () => {
|
||||
const { data } = await got(newsUrl);
|
||||
const $ = cheerio.load(data);
|
||||
const upper = $('ul.main > li.item');
|
||||
const links = upper.find('a').map((_, ele) => $(ele).attr('href'));
|
||||
const texts = upper.find('span.text').map((_, ele) => $(ele).text());
|
||||
let description = '';
|
||||
for (const [i, link] of links.entries()) {
|
||||
description += `${i + 1}、<a href="${link}">${texts[i]}</a><br>`;
|
||||
}
|
||||
description = description.replace(/<br>$/, '');
|
||||
return description;
|
||||
});
|
||||
}
|
||||
item.title = `${data.topic.content} ${dayjs(one.pubDate).format('MM月DD日')}`;
|
||||
return item;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
ctx.set('data', {
|
||||
...ctx.get('data'),
|
||||
item,
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,158 @@
|
||||
const got = require('@/utils/got');
|
||||
const cheerio = require('cheerio');
|
||||
const { parseDate } = require('@/utils/parse-date');
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const id = ctx.params.id;
|
||||
|
||||
const response = await got({
|
||||
method: 'get',
|
||||
url: `https://m.okjike.com/users/${id}`,
|
||||
headers: {
|
||||
Referer: `https://m.okjike.com/users/${id}`,
|
||||
},
|
||||
});
|
||||
|
||||
const html = response.data;
|
||||
const $ = cheerio.load(html);
|
||||
const raw = $('[type = "application/json"]').html();
|
||||
const data = JSON.parse(raw).props.pageProps;
|
||||
|
||||
const getLink = (id, type) => {
|
||||
switch (type) {
|
||||
case 'REPOST':
|
||||
return `https://m.okjike.com/reposts/${id}`;
|
||||
case 'MEDIUM':
|
||||
return `https://www.okjike.com/medium/${id}`;
|
||||
default:
|
||||
return `https://m.okjike.com/originalPosts/${id}`;
|
||||
}
|
||||
};
|
||||
|
||||
const items = await Promise.all(
|
||||
data.posts.map(async (item) => {
|
||||
const typeMap = {
|
||||
ORIGINAL_POST: '发布',
|
||||
REPOST: '转发',
|
||||
ANSWER: '回答',
|
||||
QUESTION: '提问',
|
||||
PERSONAL_UPDATE: '创建新主题',
|
||||
};
|
||||
|
||||
let linkTemplate = '';
|
||||
if (item.linkInfo && item.linkInfo.linkUrl) {
|
||||
linkTemplate = `<a href="${item.linkInfo.linkUrl}">${item.linkInfo.title}</a><br>`;
|
||||
}
|
||||
|
||||
let imgTemplate = '';
|
||||
if (item.pictures) {
|
||||
for (const picture of item.pictures) {
|
||||
imgTemplate += `<br><img src="${picture.picUrl}">`;
|
||||
}
|
||||
}
|
||||
|
||||
let content = item.content || (item.linkInfo && item.linkInfo.title) || (item.question && item.question.title) || item.title || '';
|
||||
content = content.replaceAll(/\r\n|\n|\r/g, '<br>');
|
||||
|
||||
let shortenTitle = '一条动态';
|
||||
if (content) {
|
||||
shortenTitle = content.replaceAll(/(<br>)+/g, ' ');
|
||||
content = `${content}<br><br>`;
|
||||
}
|
||||
|
||||
let repostContent;
|
||||
if (item.type === 'REPOST') {
|
||||
const screenNameTemplate = item.target.user ? `<a href="https://m.okjike.com/users/${item.target.user.username}" target="_blank">@${item.target.user.screenName}</a>` : '';
|
||||
|
||||
let repostImgTemplate = '';
|
||||
if (item.target.pictures) {
|
||||
for (const picture of item.target.pictures) {
|
||||
repostImgTemplate += `<br><img src="${picture.thumbnailUrl}">`;
|
||||
}
|
||||
}
|
||||
|
||||
repostContent = `<div class="rsshub-quote">转发 ${screenNameTemplate}: ${item.target.content}${repostImgTemplate}</div>`.replaceAll(/\r\n|\n|\r/g, '<br>');
|
||||
content = `${content}${repostContent}`;
|
||||
}
|
||||
// 部分功能未知
|
||||
/* else if (item.type === 'ANSWER') {
|
||||
let answerTextTemplate = '';
|
||||
let answerImgTemplate = '';
|
||||
let answerImgKeys = [];
|
||||
item.richtextContent.blocks &&
|
||||
item.richtextContent.blocks.forEach((item) => {
|
||||
if (item.entityRanges.length && item.text === '[图片]') {
|
||||
answerImgKeys = [...answerImgKeys, ...Object.keys(item.entityRanges)];
|
||||
} else {
|
||||
answerTextTemplate += item.text;
|
||||
}
|
||||
});
|
||||
|
||||
if (answerImgKeys.length) {
|
||||
answerImgKeys.forEach((key) => {
|
||||
const entity = item.richtextContent.entityMap[key];
|
||||
if (entity.type.toUpperCase() === 'IMAGE') {
|
||||
answerImgTemplate += `<br><img src="${entity.data.pictureUrl.middlePicUrl}">`;
|
||||
}
|
||||
});
|
||||
}
|
||||
const answerContent = `回答: ${answerTextTemplate}${answerImgTemplate}`;
|
||||
content = `${content}${answerContent}`.replace(/\n|\r/g, '<br>');
|
||||
} else if (item.type === 'QUESTION') {
|
||||
content = `在主题 <a href="https://web.okjike.com/topic/${item.topic.id}/official" target="_blank">${item.topic.content}</a> 提出了一个问题：<br><br>${content}`;
|
||||
} else if (item.type === 'PERSONAL_UPDATE') {
|
||||
shortenTitle = item.topic.content;
|
||||
content = `<img src="${item.topic.squarePicture.picUrl}"> 主题简介：<br>${item.topic.briefIntro.replace(/(?:\r\n|\r|\n)/g, '<br>')}`;
|
||||
}*/
|
||||
|
||||
const single = {
|
||||
title: `${typeMap[item.type]}了: ${shortenTitle}`,
|
||||
description: `${content}${linkTemplate}${imgTemplate}`.replace(/(<br>|\s)+$/, ''),
|
||||
pubDate: parseDate(item.createdAt),
|
||||
link: getLink(item.id, item.type),
|
||||
_extra: repostContent && {
|
||||
links: [
|
||||
{
|
||||
url: getLink(item.target.id, item.target.type),
|
||||
type: 'quote',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
if (id === 'wenhao1996' && item.topic.id === '553870e8e4b0cafb0a1bef68') {
|
||||
single.link = item.urlsInText[0].url;
|
||||
|
||||
const { data } = await got({
|
||||
method: 'get',
|
||||
url: single.link,
|
||||
headers: {
|
||||
Referer: `https://m.okjike.com/users/${id}`,
|
||||
},
|
||||
});
|
||||
const $$ = cheerio.load(data);
|
||||
$$('span.num,span.arrow').remove();
|
||||
|
||||
single.title = `一觉醒来世界发生了什么 ${$$('title').text()}`;
|
||||
|
||||
single.description = '';
|
||||
$$('div.container')
|
||||
.find('li.item')
|
||||
// eslint-disable-next-line array-callback-return
|
||||
.map((i, j) => {
|
||||
single.description += `<a href="${$$(j).find('a').attr('href')}">${$$(j).find('a').text()}</a><br>`;
|
||||
});
|
||||
}
|
||||
|
||||
return single;
|
||||
})
|
||||
);
|
||||
|
||||
ctx.state.data = {
|
||||
title: `${data.user.screenName}的即刻动态`,
|
||||
description: data.user.bio,
|
||||
link: `https://m.okjike.com/users/${id}`,
|
||||
image: data.user.avatarImage.picUrl,
|
||||
item: items,
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,198 @@
|
||||
const cheerio = require('cheerio');
|
||||
import parseDate from '@/utils/parse-date';
|
||||
import got from '@/utils/got';
|
||||
import { config } from '@/config';
|
||||
import cache from '@/utils/cache';
|
||||
|
||||
const videoAPI = 'https://api.ruguoapp.com/1.0/mediaMeta/play?type=ORIGINAL_POST';
|
||||
|
||||
module.exports = {
|
||||
topicDataHanding: (data, ctx) =>
|
||||
data.posts.map((item) => {
|
||||
let audioName, videoName, linkName;
|
||||
|
||||
// 获取纯文字内容 和 即刻原文链接
|
||||
let content, link;
|
||||
switch (item.type) {
|
||||
case 'ORIGINAL_POST':
|
||||
content = item.content;
|
||||
link = `https://m.okjike.com/originalPosts/${item.id}`;
|
||||
break;
|
||||
|
||||
/* case 'QUESTION':
|
||||
content = item.title;
|
||||
link = `https://m.okjike.com/questions/${item.id}`;
|
||||
break;
|
||||
case 'OFFICIAL_MESSAGE':
|
||||
content = item.content;
|
||||
link = `https://web.okjike.com/message-detail/${item.id}/officialMessage`;
|
||||
break;*/
|
||||
default:
|
||||
content = '未知类型，请前往GitHub提交issue';
|
||||
link = 'https://github.com/DIYgod/RSSHub/issues';
|
||||
}
|
||||
|
||||
// rss内容
|
||||
let description = '';
|
||||
// 作者昵称
|
||||
let author = '';
|
||||
|
||||
// 添加内容作者信息
|
||||
if (item.user) {
|
||||
author = item.user.screenName;
|
||||
if (ctx.req.param('showUid')) {
|
||||
description += `<span>用户昵称：${author} <br> Username：${item.user.username}</span><br>`;
|
||||
}
|
||||
}
|
||||
|
||||
if (item.linkInfo) {
|
||||
const linkUrl = item.linkInfo.originalLinkUrl || item.linkInfo.linkUrl;
|
||||
|
||||
// 对于即刻抓取的微信公众号文章 特殊处理
|
||||
// 此时 Rss原文链接 变为 微信公众号链接
|
||||
if (new URL(linkUrl).host === 'mp.weixin.qq.com') {
|
||||
link = linkUrl;
|
||||
}
|
||||
|
||||
// 1. 音频
|
||||
const audioObject = item.linkInfo.audio || item.audio;
|
||||
if (audioObject) {
|
||||
const audioImage = audioObject.image.picUrl || audioObject.image.thumbnailUrl;
|
||||
const audioLink = linkUrl;
|
||||
const audioTitle = audioObject.title;
|
||||
const audioAuthor = audioObject.author;
|
||||
audioName = `${audioTitle} - ${audioAuthor}`;
|
||||
description += `
|
||||
<img src="${audioImage}">
|
||||
<a href="${audioLink}">${audioName}</a>
|
||||
`;
|
||||
}
|
||||
|
||||
// 2. 视频
|
||||
const videoObject = item.linkInfo.video || item.video;
|
||||
if (videoObject) {
|
||||
const videoImage = videoObject.image.picUrl || videoObject.image.thumbnailUrl;
|
||||
const videoLink = linkUrl;
|
||||
const videoDuration = Math.floor(videoObject.duration / 60000);
|
||||
videoName = item.linkInfo.title;
|
||||
description += `
|
||||
<img src="${videoImage}">
|
||||
<a href="${videoLink}">${videoName || '观看视频'} - 约${videoDuration}分钟</a>
|
||||
`;
|
||||
}
|
||||
|
||||
// 3. 链接
|
||||
if (!audioObject && !videoObject && linkUrl) {
|
||||
// 部分链接有标题
|
||||
linkName = item.linkInfo.title;
|
||||
const linkTitle = linkName || '访问原文';
|
||||
// 部分链接有缩略图
|
||||
const linkImage = item.linkInfo.pictureUrl;
|
||||
const imageTag = `<img src="${linkImage}">`;
|
||||
description += `
|
||||
${linkImage ? imageTag : ''}
|
||||
<a href="${linkUrl}">${linkTitle}</a>
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. 文字内容
|
||||
description += description ? `<br>${content}` : content;
|
||||
|
||||
// 5. 图片
|
||||
if (item.pictures) {
|
||||
for (const pic of item.pictures) {
|
||||
if (pic.format === 'gif') {
|
||||
description += `<img src="${pic.picUrl.split('?imageMogr2/')[0]}">`;
|
||||
} else {
|
||||
// jpeg, bmp, png, gif, webp
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types
|
||||
// let type = 'jpeg';
|
||||
// switch (pic.format) {
|
||||
// case 'bmp':
|
||||
// type = 'bmp';
|
||||
// break;
|
||||
// case 'png':
|
||||
// type = 'png';
|
||||
// break;
|
||||
// default:
|
||||
// break;
|
||||
// }
|
||||
const imgUrl = /\.[\da-z]+?\?imageMogr2/.test(pic.picUrl) ? pic.picUrl.split('?imageMogr2/')[0] : pic.picUrl.replace(/thumbnail\/.+/, '');
|
||||
description += `<br><img src="${imgUrl}">`;
|
||||
// description += `<br><picture><source srcset="${
|
||||
// pic.picUrl.split('/thumbnail/')[0]
|
||||
// }/strip/format/webp" type="image/webp"><source srcset="${imgUrl}" type="image/${type}"><img src="${imgUrl}"></picture>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 6. 视频
|
||||
if (item.video) {
|
||||
description += `<br><video src="${item.video.url}" controls></video>`;
|
||||
}
|
||||
|
||||
// rss标题
|
||||
// 优先将音频和视频名作为标题
|
||||
// 其次将正文内容作为标题
|
||||
// 若都没有 则是推送型消息，将连接标题作为主题
|
||||
// “无题” fallback
|
||||
const title = audioName || videoName || content || linkName || '无题';
|
||||
|
||||
return {
|
||||
title,
|
||||
description: description.trim().replaceAll('\n', '<br>'),
|
||||
pubDate: parseDate.parse(item.createdAt),
|
||||
author,
|
||||
link,
|
||||
};
|
||||
}),
|
||||
constructTopicEntry: async (ctx, url) => {
|
||||
const data = await cache.tryGet(
|
||||
url,
|
||||
async () => {
|
||||
const resp = await got(url);
|
||||
const html = resp.data;
|
||||
const $ = cheerio.load(html);
|
||||
const raw = $('[type = "application/json"]').html();
|
||||
const data = JSON.parse(raw).props.pageProps;
|
||||
data.posts = await Promise.all(
|
||||
data.posts.map(async (item) => {
|
||||
if (!item.video) {
|
||||
return item;
|
||||
}
|
||||
|
||||
const videoUrl = `${videoAPI}&id=${item.id}`;
|
||||
const videoRes = await got(videoUrl);
|
||||
item.video = videoRes.data;
|
||||
|
||||
return item;
|
||||
})
|
||||
);
|
||||
|
||||
return data;
|
||||
},
|
||||
false,
|
||||
config.cache.routeExpire
|
||||
);
|
||||
|
||||
if (data.length === 0) {
|
||||
ctx.set('data', {
|
||||
title: '主题 ID 不存在，或该主题暂无内容',
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
const topic = data.topic;
|
||||
|
||||
ctx.set('data', {
|
||||
title: `${topic.content} - 即刻圈子`,
|
||||
link: url,
|
||||
description: topic.briefIntro,
|
||||
image: topic.squarePicture.picUrl || topic.squarePicture.middlePicUrl || topic.squarePicture.thumbnailUrl,
|
||||
// item: topicDataHanding(data),
|
||||
});
|
||||
|
||||
return data;
|
||||
},
|
||||
};
|
||||
@@ -43,4 +43,4 @@ const json = (data) => {
|
||||
return JSON.stringify(jsonFeed, null, 4);
|
||||
};
|
||||
|
||||
module.exports = json;
|
||||
export default json;
|
||||
@@ -1,4 +1,4 @@
|
||||
const dayjs = require('dayjs');
|
||||
import dayjs from 'dayjs';
|
||||
|
||||
/**
|
||||
* This function should be used by RSSHub middleware only.
|
||||
@@ -59,4 +59,4 @@ function getOwnershipFieldFromURL(item) {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = rss3Ums;
|
||||
export default rss3Ums;
|
||||
+5
-11
@@ -23,7 +23,7 @@
|
||||
"build:all": "npm run build:radar && npm run build:maintainer",
|
||||
"build:maintainer": "node scripts/workflow/build-maintainer.js",
|
||||
"build:radar": "node scripts/workflow/build-radar.js",
|
||||
"dev": "cross-env NODE_ENV=dev nodemon --inspect lib/index.js",
|
||||
"dev": "cross-env NODE_ENV=dev tsx watch lib/index.ts",
|
||||
"format": "eslint --cache --fix \"**/*.{js,yml}\" && node website/docs/.format/format.mjs && prettier \"**/*.{js,json}\" --write",
|
||||
"format:check": "eslint --cache \"**/*.{js,yml}\" && prettier \"**/*.{js,json}\" --check",
|
||||
"format:staged": "lint-staged",
|
||||
@@ -44,14 +44,6 @@
|
||||
"*.md?(x)": "node website/docs/.format/format.mjs --staged",
|
||||
"*.yml": "eslint --cache --fix"
|
||||
},
|
||||
"nodemonConfig": {
|
||||
"ext": "art,js,mjs,json",
|
||||
"watch": [
|
||||
".env",
|
||||
"lib",
|
||||
"test"
|
||||
]
|
||||
},
|
||||
"jest": {
|
||||
"bail": true,
|
||||
"collectCoverageFrom": [
|
||||
@@ -83,11 +75,13 @@
|
||||
"outputDirectory": "coverage"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "1.4.0",
|
||||
"@koa/router": "12.0.1",
|
||||
"@notionhq/client": "2.2.14",
|
||||
"@postlight/parser": "2.2.3",
|
||||
"@sentry/node": "7.94.1",
|
||||
"@tonyrl/rand-user-agent": "2.0.46",
|
||||
"@types/node": "20.11.5",
|
||||
"aes-js": "3.1.2",
|
||||
"art-template": "4.13.2",
|
||||
"bbcodejs": "0.0.4",
|
||||
@@ -104,6 +98,7 @@
|
||||
"git-rev-sync": "3.0.2",
|
||||
"googleapis": "131.0.0",
|
||||
"got": "11.8.6",
|
||||
"hono": "3.12.6",
|
||||
"html-to-text": "9.0.5",
|
||||
"https-proxy-agent": "7.0.2",
|
||||
"iconv-lite": "0.6.3",
|
||||
@@ -176,7 +171,6 @@
|
||||
"@types/mailparser": "3.4.4",
|
||||
"@types/markdown-it": "13.0.7",
|
||||
"@types/module-alias": "2.0.4",
|
||||
"@types/nodemon": "1.19.6",
|
||||
"@types/plist": "3.0.5",
|
||||
"@types/request-promise-native": "1.0.21",
|
||||
"@types/require-all": "3.0.6",
|
||||
@@ -199,7 +193,6 @@
|
||||
"lint-staged": "15.2.0",
|
||||
"mockdate": "3.0.5",
|
||||
"nock": "13.5.0",
|
||||
"nodemon": "3.0.3",
|
||||
"prettier": "3.2.4",
|
||||
"remark": "14.0.3",
|
||||
"remark-custom-heading-id": "1.0.1",
|
||||
@@ -216,6 +209,7 @@
|
||||
"supertest": "6.3.4",
|
||||
"to-vfile": "8.0.0",
|
||||
"tosource": "2.0.0-alpha.3",
|
||||
"tsx": "4.7.0",
|
||||
"unified": "9.2.2",
|
||||
"unist-util-visit": "4.1.2",
|
||||
"unist-util-visit-parents": "5.1.3"
|
||||
|
||||
Generated
+368
-183
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": ["ES2022"],
|
||||
"strict": true,
|
||||
"jsx": "react-jsx",
|
||||
"jsxImportSource": "hono/jsx",
|
||||
"paths": {
|
||||
"@/*": ["./lib/*"]
|
||||
},
|
||||
"esModuleInterop": true
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user