feat: replace co-redis and redis with ioredis, make cache module clearer

2026-09-24 23:12:34 +08:00 · 2021-01-27 19:16:59 +08:00
parent 9d3661c984
commit 97893f2701
6 changed files with 70 additions and 114 deletions
@@ -347,8 +347,6 @@ RSSHub supports two caching methods: memory and redis

 `REDIS_URL`: Redis target address（invalid when `CACHE_TYPE` is set to memory）, default to `redis://localhost:6379/`

-`REDIS_PASSWORD`: Redis password（invalid when `CACHE_TYPE` is set to memory)
-
 ### Proxy Configurations

 Partial routes have a strict anti-crawler policy, and can be configured to use proxy
@@ -351,8 +351,6 @@ RSSHub 支持 `memory` 和 `redis` 两种缓存方式

 `REDIS_URL`: Redis 连接地址（redis 缓存类型时有效），默认为 `redis://localhost:6379/`

-`REDIS_PASSWORD`: Redis 连接密码（redis 缓存类型时有效）
-
 ### 代理配置

 部分路由反爬严格，可以配置使用代理抓取。
@@ -45,10 +45,6 @@ const calculateValue = () => {
        titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
        redis: {
            url: envs.REDIS_URL || 'redis://localhost:6379/',
-            options: {
-                // 支持这些参数 https://github.com/NodeRedis/node_redis#options-object-properties
-                password: envs.REDIS_PASSWORD || null,
-            },
        },
        nodeName: envs.NODE_NAME,
        suffix: envs.SUFFIX,
@@ -1,25 +1,19 @@
-const wrapper = require('co-redis');
-const Redis = require('redis');
+const Redis = require('ioredis');
 const Lru = require('lru-cache');
 const md5 = require('@/utils/md5');
 const config = require('@/config').value;
 const logger = require('@/utils/logger');

-module.exports = function (app, options = {}) {
+module.exports = function (app) {
    let available = false;
-
-    const { prefix = 'koa-redis-cache:', expire = config.cache.routeExpire, passParam = '', maxLength = Infinity, ignoreQuery = true } = options;
-
    const globalCache = {
        get: null,
        set: null,
    };
+
    if (config.cache.type === 'redis') {
-        const { host: redisHost = 'localhost', port: redisPort = 6379, url: redisUrl = `redis://${redisHost}:${redisPort}/`, options: redisOptions = {} } = config.redis || {};
-        if (!redisOptions.password) {
-            delete redisOptions.password;
-        }
-        const redisClient = wrapper(Redis.createClient(redisUrl, redisOptions));
+        const redisClient = new Redis(config.redis.url);
+
        redisClient.on('error', (error) => {
            available = false;
            logger.error('Redis error: ', error);
@@ -54,7 +48,7 @@ module.exports = function (app, options = {}) {
                    value = JSON.stringify(value);
                }
                if (key) {
-                    redisClient.setex(key, maxAge, value);
+                    redisClient.set(key, value, 'NX', 'EX', maxAge);
                }
            },
            client: redisClient,
@@ -69,13 +63,13 @@ module.exports = function (app, options = {}) {
        globalCache.set = app.context.cache.set;
    } else if (config.cache.type === 'memory') {
        const pageCache = new Lru({
-            maxAge: expire * 1000,
-            max: maxLength,
+            maxAge: config.cache.routeExpire * 1000,
+            max: Infinity,
        });

        const routeCache = new Lru({
-            maxAge: expire * 1000,
-            max: maxLength,
+            maxAge: config.cache.routeExpire * 1000,
+            max: Infinity,
            updateAgeOnGet: true,
        });

@@ -147,66 +141,40 @@ module.exports = function (app, options = {}) {
        return v;
    };

-    async function getCache(ctx, key) {
-        const value = await globalCache.get(key);
-        let ok = false;
-
-        if (value) {
-            ctx.response.status = 200;
-            if (config.cache.type === 'redis') {
-                ctx.response.set({
-                    'X-Koa-Redis-Cache': 'true',
-                });
-            } else if (config.cache.type === 'memory') {
-                ctx.response.set({
-                    'X-Koa-Memory-Cache': 'true',
-                });
-            }
-            try {
-                ctx.state.data = JSON.parse(value);
-            } catch (e) {
-                ctx.state.data = {};
-            }
-            ok = true;
-        }
-
-        return ok;
-    }
-
-    async function setCache(ctx, key, expire) {
-        if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) {
-            ctx.state.data.lastBuildDate = new Date().toUTCString();
-            const body = JSON.stringify(ctx.state.data);
-            await globalCache.set(key, body, expire);
-        }
-    }
-
    return async function cache(ctx, next) {
-        const { url, path } = ctx.request;
-        const resolvedPrefix = typeof prefix === 'function' ? prefix.call(ctx, ctx) : prefix;
-        const key = resolvedPrefix + md5(ignoreQuery ? path : url);
+        const key = 'koa-redis-cache:' + md5(ctx.request.path);

-        if (!available || (passParam && ctx.request.query[passParam])) {
+        if (!available) {
            return await next();
        }

-        let ok = false;
        try {
-            ok = await getCache(ctx, key);
+            const value = await globalCache.get(key);
+
+            if (value) {
+                ctx.response.status = 200;
+                if (config.cache.type === 'redis') {
+                    ctx.response.set({
+                        'X-Koa-Redis-Cache': 'true',
+                    });
+                } else if (config.cache.type === 'memory') {
+                    ctx.response.set({
+                        'X-Koa-Memory-Cache': 'true',
+                    });
+                }
+                ctx.state.data = JSON.parse(value);
+                return;
+            }
        } catch (e) {
-            ok = false;
-        }
-        if (ok) {
-            return;
+            //
        }

        await next();

-        try {
-            const trueExpire = expire;
-            setCache(ctx, key, trueExpire);
-        } catch (e) {
-            //
+        if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) {
+            ctx.state.data.lastBuildDate = new Date().toUTCString();
+            const body = JSON.stringify(ctx.state.data);
+            await globalCache.set(key, body, config.cache.routeExpire);
        }
    };
 };
@@ -72,7 +72,6 @@
    "bbcodejs": "0.0.4",
    "cheerio": "1.0.0-rc.3",
    "chrono-node": "2.1.10",
-    "co-redis": "2.1.1",
    "crypto-js": "4.0.0",
    "currency-symbol-map": "4.0.4",
    "dayjs": "1.10.3",
@@ -87,6 +86,7 @@
    "https-proxy-agent": "5.0.0",
    "iconv-lite": "0.6.2",
    "instagram-private-api": "1.43.3",
+    "ioredis": "4.19.4",
    "ip-regex": "4.2.0",
    "is-localhost-ip": "1.4.0",
    "jsdom": "16.4.0",
@@ -107,7 +107,6 @@
    "plist": "3.0.1",
    "puppeteer": "5.5.0",
    "query-string": "6.13.8",
-    "redis": "3.0.2",
    "require-all": "3.0.0",
    "rss-parser": "3.10.0",
    "showdown": "1.9.1",
@@ -3586,12 +3586,10 @@ cloudscraper@^4.6.0:
  dependencies:
    request-promise "^4.2.4"

-co-redis@2.1.1:
-  version "2.1.1"
-  resolved "https://registry.yarnpkg.com/co-redis/-/co-redis-2.1.1.tgz#8bc3717977c9ee5bfb6b28f39ae01bcf724d2dde"
-  integrity sha1-i8NxeXfJ7lv7ayjzmuAbz3JNLd4=
-  dependencies:
-    es6-promisify "^4.1.0"
+cluster-key-slot@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz#30474b2a981fb12172695833052bc0d01336d10d"
+  integrity sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==

 co@3.1.0:
  version "3.1.0"
@@ -4492,10 +4490,10 @@ delegates@^1.0.0:
  resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a"
  integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=

-denque@^1.4.1:
-  version "1.4.1"
-  resolved "https://registry.yarnpkg.com/denque/-/denque-1.4.1.tgz#6744ff7641c148c3f8a69c307e51235c1f4a37cf"
-  integrity sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==
+denque@^1.1.0:
+  version "1.5.0"
+  resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.0.tgz#773de0686ff2d8ec2ff92914316a47b73b1c73de"
+  integrity sha512-CYiCSgIF1p6EUByQPlGkKnP1M9g0ZV3qMIrqMqZqdwazygIA/YP2vrbcyl1h/WppKJTdl1F85cXIle+394iDAQ==

 depd@^2.0.0, depd@~2.0.0:
  version "2.0.0"
@@ -5005,23 +5003,11 @@ es-to-primitive@^1.2.0:
    is-date-object "^1.0.1"
    is-symbol "^1.0.2"

-es6-promise@^3.2.1:
-  version "3.3.1"
-  resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-3.3.1.tgz#a08cdde84ccdbf34d027a1451bc91d4bcd28a613"
-  integrity sha1-oIzd6EzNvzTQJ6FFG8kdS80ophM=
-
 es6-promise@^4.1.0:
  version "4.2.6"
  resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.2.6.tgz#b685edd8258886365ea62b57d30de28fadcd974f"
  integrity sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q==

-es6-promisify@^4.1.0:
-  version "4.1.0"
-  resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-4.1.0.tgz#dbcc1a3e93fdb8e9e709744182b7d2080c03fdb3"
-  integrity sha1-28waPpP9uOnnCXRBgrfSCAwD/bM=
-  dependencies:
-    es6-promise "^3.2.1"
-
 escape-goat@^2.0.0:
  version "2.1.1"
  resolved "https://registry.yarnpkg.com/escape-goat/-/escape-goat-2.1.1.tgz#1b2dc77003676c457ec760b2dc68edb648188675"
@@ -6824,6 +6810,22 @@ invert-kv@^2.0.0:
  resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02"
  integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA==

+ioredis@4.19.4:
+  version "4.19.4"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.19.4.tgz#11112005f87ad3acac247ada3b22eb31b947f7c7"
+  integrity sha512-3haQWw9dpEjcfVcRktXlayVNrrqvvc2io7Q/uiV2UsYw8/HC2YwwJr78Wql7zu5bzwci0x9bZYA69U7KkevAvw==
+  dependencies:
+    cluster-key-slot "^1.1.0"
+    debug "^4.1.1"
+    denque "^1.1.0"
+    lodash.defaults "^4.2.0"
+    lodash.flatten "^4.4.0"
+    p-map "^2.1.0"
+    redis-commands "1.6.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.0.1"
+
 ip-regex@4.2.0, ip-regex@^4.1.0:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-4.2.0.tgz#a03f5eb661d9a154e3973a03de8b23dd0ad6892e"
@@ -8245,7 +8247,7 @@ lodash.debounce@^4.0.8:
  resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
  integrity sha1-gteb/zCmfEAF/9XiUVMArZyk168=

-lodash.defaults@^4.0.1:
+lodash.defaults@^4.0.1, lodash.defaults@^4.2.0:
  version "4.2.0"
  resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c"
  integrity sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=
@@ -8255,7 +8257,7 @@ lodash.filter@^4.4.0:
  resolved "https://registry.yarnpkg.com/lodash.filter/-/lodash.filter-4.6.0.tgz#668b1d4981603ae1cc5a6fa760143e480b4c4ace"
  integrity sha1-ZosdSYFgOuHMWm+nYBQ+SAtMSs4=

-lodash.flatten@^4.2.0:
+lodash.flatten@^4.2.0, lodash.flatten@^4.4.0:
  version "4.4.0"
  resolved "https://registry.yarnpkg.com/lodash.flatten/-/lodash.flatten-4.4.0.tgz#f31c22225a9632d2bbf8e4addbef240aa765a61f"
  integrity sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=
@@ -9613,7 +9615,7 @@ p-locate@^4.1.0:
  dependencies:
    p-limit "^2.2.0"

-p-map@^2.0.0:
+p-map@^2.0.0, p-map@^2.1.0:
  version "2.1.0"
  resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
  integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
@@ -10817,10 +10819,10 @@ rechoir@^0.6.2:
  dependencies:
    resolve "^1.1.6"

-redis-commands@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.5.0.tgz#80d2e20698fe688f227127ff9e5164a7dd17e785"
-  integrity sha512-6KxamqpZ468MeQC3bkWmCB1fp56XL64D4Kf0zJSwDZbVLLm7KFkoIcHrgRvQ+sk8dnhySs7+yBg94yIkAK7aJg==
+redis-commands@1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.6.0.tgz#36d4ca42ae9ed29815cdb30ad9f97982eba1ce23"
+  integrity sha512-2jnZ0IkjZxvguITjFTrGiLyzQZcTvaw8DAaCXxZq/dsHXz7KfMQ3OUJy7Tz9vnRtZRVz6VRCPDvruvU8Ts44wQ==

 redis-errors@^1.0.0, redis-errors@^1.2.0:
  version "1.2.0"
@@ -10834,16 +10836,6 @@ redis-parser@^3.0.0:
  dependencies:
    redis-errors "^1.0.0"

-redis@3.0.2:
-  version "3.0.2"
-  resolved "https://registry.yarnpkg.com/redis/-/redis-3.0.2.tgz#bd47067b8a4a3e6a2e556e57f71cc82c7360150a"
-  integrity sha512-PNhLCrjU6vKVuMOyFu7oSP296mwBkcE6lrAjruBYG5LgdSqtRBoVQIylrMyVZD/lkF24RSNNatzvYag6HRBHjQ==
-  dependencies:
-    denque "^1.4.1"
-    redis-commands "^1.5.0"
-    redis-errors "^1.2.0"
-    redis-parser "^3.0.0"
-
 reduce@^1.0.1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/reduce/-/reduce-1.0.2.tgz#0cd680ad3ffe0b060e57a5c68bdfce37168d361b"
@@ -11879,6 +11871,11 @@ staged-git-files@1.2.0:
  resolved "https://registry.yarnpkg.com/staged-git-files/-/staged-git-files-1.2.0.tgz#05b43f4fcec6d4799ac41c826ecd17c0bd53ce82"
  integrity sha512-MYK3aDsO8XAXkv2ASsrznObxVDlocYm7gc/cMk/hB4vbJZeEqOO7H1mZR7EY2C5q3YgKOBo3Tmu0D30h7RjdWg==

+standard-as-callback@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.0.1.tgz#ed8bb25648e15831759b6023bdb87e6b60b38126"
+  integrity sha512-NQOxSeB8gOI5WjSaxjBgog2QFw55FV8TkS6Y07BiB3VJ8xNTvUYm0wl0s8ObgQ5NhdpnNfigMIKjgPESzgr4tg==
+
 state-toggle@^1.0.0:
  version "1.0.3"
  resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe"