From 97893f270110b751e3ec668f37cb291f249b3dbf Mon Sep 17 00:00:00 2001 From: DIYgod Date: Wed, 27 Jan 2021 19:16:59 +0800 Subject: [PATCH] feat: replace co-redis and redis with ioredis, make cache module clearer --- docs/en/install/README.md | 2 - docs/install/README.md | 2 - lib/config.js | 4 -- lib/middleware/cache.js | 98 +++++++++++++-------------------------- package.json | 3 +- yarn.lock | 75 ++++++++++++++---------------- 6 files changed, 70 insertions(+), 114 deletions(-) diff --git a/docs/en/install/README.md b/docs/en/install/README.md index 976d0fe3fa..cfd16b3916 100644 --- a/docs/en/install/README.md +++ b/docs/en/install/README.md @@ -347,8 +347,6 @@ RSSHub supports two caching methods: memory and redis `REDIS_URL`: Redis target address（invalid when `CACHE_TYPE` is set to memory）, default to `redis://localhost:6379/` -`REDIS_PASSWORD`: Redis password（invalid when `CACHE_TYPE` is set to memory) - ### Proxy Configurations Partial routes have a strict anti-crawler policy, and can be configured to use proxy diff --git a/docs/install/README.md b/docs/install/README.md index 8e40392500..fd1e6dd674 100644 --- a/docs/install/README.md +++ b/docs/install/README.md @@ -351,8 +351,6 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式 `REDIS_URL`: Redis 连接地址（redis 缓存类型时有效），默认为 `redis://localhost:6379/` -`REDIS_PASSWORD`: Redis 连接密码（redis 缓存类型时有效） - ### 代理配置 部分路由反爬严格，可以配置使用代理抓取。 diff --git a/lib/config.js b/lib/config.js index 7512fe9eb3..5ae487c815 100644 --- a/lib/config.js +++ b/lib/config.js @@ -45,10 +45,6 @@ const calculateValue = () => { titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150, redis: { url: envs.REDIS_URL || 'redis://localhost:6379/', - options: { - // 支持这些参数 https://github.com/NodeRedis/node_redis#options-object-properties - password: envs.REDIS_PASSWORD || null, - }, }, nodeName: envs.NODE_NAME, suffix: envs.SUFFIX, diff --git a/lib/middleware/cache.js b/lib/middleware/cache.js index 1500f89c57..7120486759 100644 --- a/lib/middleware/cache.js +++ b/lib/middleware/cache.js @@ -1,25 +1,19 @@ -const wrapper = require('co-redis'); -const Redis = require('redis'); +const Redis = require('ioredis'); const Lru = require('lru-cache'); const md5 = require('@/utils/md5'); const config = require('@/config').value; const logger = require('@/utils/logger'); -module.exports = function (app, options = {}) { +module.exports = function (app) { let available = false; - - const { prefix = 'koa-redis-cache:', expire = config.cache.routeExpire, passParam = '', maxLength = Infinity, ignoreQuery = true } = options; - const globalCache = { get: null, set: null, }; + if (config.cache.type === 'redis') { - const { host: redisHost = 'localhost', port: redisPort = 6379, url: redisUrl = `redis://${redisHost}:${redisPort}/`, options: redisOptions = {} } = config.redis || {}; - if (!redisOptions.password) { - delete redisOptions.password; - } - const redisClient = wrapper(Redis.createClient(redisUrl, redisOptions)); + const redisClient = new Redis(config.redis.url); + redisClient.on('error', (error) => { available = false; logger.error('Redis error: ', error); @@ -54,7 +48,7 @@ module.exports = function (app, options = {}) { value = JSON.stringify(value); } if (key) { - redisClient.setex(key, maxAge, value); + redisClient.set(key, value, 'NX', 'EX', maxAge); } }, client: redisClient, @@ -69,13 +63,13 @@ module.exports = function (app, options = {}) { globalCache.set = app.context.cache.set; } else if (config.cache.type === 'memory') { const pageCache = new Lru({ - maxAge: expire * 1000, - max: maxLength, + maxAge: config.cache.routeExpire * 1000, + max: Infinity, }); const routeCache = new Lru({ - maxAge: expire * 1000, - max: maxLength, + maxAge: config.cache.routeExpire * 1000, + max: Infinity, updateAgeOnGet: true, }); @@ -147,66 +141,40 @@ module.exports = function (app, options = {}) { return v; }; - async function getCache(ctx, key) { - const value = await globalCache.get(key); - let ok = false; - - if (value) { - ctx.response.status = 200; - if (config.cache.type === 'redis') { - ctx.response.set({ - 'X-Koa-Redis-Cache': 'true', - }); - } else if (config.cache.type === 'memory') { - ctx.response.set({ - 'X-Koa-Memory-Cache': 'true', - }); - } - try { - ctx.state.data = JSON.parse(value); - } catch (e) { - ctx.state.data = {}; - } - ok = true; - } - - return ok; - } - - async function setCache(ctx, key, expire) { - if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) { - ctx.state.data.lastBuildDate = new Date().toUTCString(); - const body = JSON.stringify(ctx.state.data); - await globalCache.set(key, body, expire); - } - } - return async function cache(ctx, next) { - const { url, path } = ctx.request; - const resolvedPrefix = typeof prefix === 'function' ? prefix.call(ctx, ctx) : prefix; - const key = resolvedPrefix + md5(ignoreQuery ? path : url); + const key = 'koa-redis-cache:' + md5(ctx.request.path); - if (!available || (passParam && ctx.request.query[passParam])) { + if (!available) { return await next(); } - let ok = false; try { - ok = await getCache(ctx, key); + const value = await globalCache.get(key); + + if (value) { + ctx.response.status = 200; + if (config.cache.type === 'redis') { + ctx.response.set({ + 'X-Koa-Redis-Cache': 'true', + }); + } else if (config.cache.type === 'memory') { + ctx.response.set({ + 'X-Koa-Memory-Cache': 'true', + }); + } + ctx.state.data = JSON.parse(value); + return; + } } catch (e) { - ok = false; - } - if (ok) { - return; + // } await next(); - try { - const trueExpire = expire; - setCache(ctx, key, trueExpire); - } catch (e) { - // + if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) { + ctx.state.data.lastBuildDate = new Date().toUTCString(); + const body = JSON.stringify(ctx.state.data); + await globalCache.set(key, body, config.cache.routeExpire); } }; }; diff --git a/package.json b/package.json index 447abd8364..e0c4766045 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,6 @@ "bbcodejs": "0.0.4", "cheerio": "1.0.0-rc.3", "chrono-node": "2.1.10", - "co-redis": "2.1.1", "crypto-js": "4.0.0", "currency-symbol-map": "4.0.4", "dayjs": "1.10.3", @@ -87,6 +86,7 @@ "https-proxy-agent": "5.0.0", "iconv-lite": "0.6.2", "instagram-private-api": "1.43.3", + "ioredis": "4.19.4", "ip-regex": "4.2.0", "is-localhost-ip": "1.4.0", "jsdom": "16.4.0", @@ -107,7 +107,6 @@ "plist": "3.0.1", "puppeteer": "5.5.0", "query-string": "6.13.8", - "redis": "3.0.2", "require-all": "3.0.0", "rss-parser": "3.10.0", "showdown": "1.9.1", diff --git a/yarn.lock b/yarn.lock index e99293b6d7..9a48786b08 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3586,12 +3586,10 @@ cloudscraper@^4.6.0: dependencies: request-promise "^4.2.4" -co-redis@2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/co-redis/-/co-redis-2.1.1.tgz#8bc3717977c9ee5bfb6b28f39ae01bcf724d2dde" - integrity sha1-i8NxeXfJ7lv7ayjzmuAbz3JNLd4= - dependencies: - es6-promisify "^4.1.0" +cluster-key-slot@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d" + integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw== co@3.1.0: version "3.1.0" @@ -4492,10 +4490,10 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= -denque@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/denque/-/denque-1.4.1.tgz#6744ff7641c148c3f8a69c307e51235c1f4a37cf" - integrity sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ== +denque@^1.1.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de" + integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ== depd@^2.0.0, depd@~2.0.0: version "2.0.0" @@ -5005,23 +5003,11 @@ es-to-primitive@^1.2.0: is-date-object "^1.0.1" is-symbol "^1.0.2" -es6-promise@^3.2.1: - version "3.3.1" - resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.3.1.tgz#a08cdde84ccdbf34d027a1451bc91d4bcd28a613" - integrity sha1-oIzd6EzNvzTQJ6FFG8kdS80ophM= - es6-promise@^4.1.0: version "4.2.6" resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.6.tgz#b685edd8258886365ea62b57d30de28fadcd974f" integrity sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q== -es6-promisify@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-4.1.0.tgz#dbcc1a3e93fdb8e9e709744182b7d2080c03fdb3" - integrity sha1-28waPpP9uOnnCXRBgrfSCAwD/bM= - dependencies: - es6-promise "^3.2.1" - escape-goat@^2.0.0: version "2.1.1" resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675" @@ -6824,6 +6810,22 @@ invert-kv@^2.0.0: resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02" integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA== +ioredis@4.19.4: + version "4.19.4" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.19.4.tgz#11112005f87ad3acac247ada3b22eb31b947f7c7" + integrity sha512-3haQWw9dpEjcfVcRktXlayVNrrqvvc2io7Q/uiV2UsYw8/HC2YwwJr78Wql7zu5bzwci0x9bZYA69U7KkevAvw== + dependencies: + cluster-key-slot "^1.1.0" + debug "^4.1.1" + denque "^1.1.0" + lodash.defaults "^4.2.0" + lodash.flatten "^4.4.0" + p-map "^2.1.0" + redis-commands "1.6.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.0.1" + ip-regex@4.2.0, ip-regex@^4.1.0: version "4.2.0" resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.2.0.tgz#a03f5eb661d9a154e3973a03de8b23dd0ad6892e" @@ -8245,7 +8247,7 @@ lodash.debounce@^4.0.8: resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168= -lodash.defaults@^4.0.1: +lodash.defaults@^4.0.1, lodash.defaults@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw= @@ -8255,7 +8257,7 @@ lodash.filter@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace" integrity sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4= -lodash.flatten@^4.2.0: +lodash.flatten@^4.2.0, lodash.flatten@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f" integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8= @@ -9613,7 +9615,7 @@ p-locate@^4.1.0: dependencies: p-limit "^2.2.0" -p-map@^2.0.0: +p-map@^2.0.0, p-map@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175" integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw== @@ -10817,10 +10819,10 @@ rechoir@^0.6.2: dependencies: resolve "^1.1.6" -redis-commands@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.5.0.tgz#80d2e20698fe688f227127ff9e5164a7dd17e785" - integrity sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg== +redis-commands@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.6.0.tgz#36d4ca42ae9ed29815cdb30ad9f97982eba1ce23" + integrity sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ== redis-errors@^1.0.0, redis-errors@^1.2.0: version "1.2.0" @@ -10834,16 +10836,6 @@ redis-parser@^3.0.0: dependencies: redis-errors "^1.0.0" -redis@3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/redis/-/redis-3.0.2.tgz#bd47067b8a4a3e6a2e556e57f71cc82c7360150a" - integrity sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ== - dependencies: - denque "^1.4.1" - redis-commands "^1.5.0" - redis-errors "^1.2.0" - redis-parser "^3.0.0" - reduce@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/reduce/-/reduce-1.0.2.tgz#0cd680ad3ffe0b060e57a5c68bdfce37168d361b" @@ -11879,6 +11871,11 @@ staged-git-files@1.2.0: resolved "https://registry.yarnpkg.com/staged-git-files/-/staged-git-files-1.2.0.tgz#05b43f4fcec6d4799ac41c826ecd17c0bd53ce82" integrity sha512-MYK3aDsO8XAXkv2ASsrznObxVDlocYm7gc/cMk/hB4vbJZeEqOO7H1mZR7EY2C5q3YgKOBo3Tmu0D30h7RjdWg== +standard-as-callback@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.0.1.tgz#ed8bb25648e15831759b6023bdb87e6b60b38126" + integrity sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg== + state-toggle@^1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe"