feat(route): Telegram Channels (with video and files) (#13255)

* add tg via client library for full-res video + photo support

* fix expiring links

* fix photo downloads

* add inline thumbnails

* document thumbnails + streaming cache lock workaround

* true streaming support via Range requests

* add radar rules

* parse entities in messages to html

* tune chunking for video downloads

* add CLI option to generate TG_SESSION

* update pnpm-lock.yaml

* update telegram

* fix console usages

* remove unnecessary comment

* add english docs

* respond to review comments

* update pnpm-lock.yaml

* fix linting

* backwards compatibility for routeParams

* fix linting

* tune requestSize for smoother streaming

* fix incorrect tail post handling

* fix apiId env var bug

* html.unparse must be called on stripped text, not source markdown

* fix video streaming (requestSize is raw TL `limit`, chunkSize auto-aligns offsets by gramjs)

* replace newlines with <br> in messages because unparse doesn't do that

* must be replaceAll \n to <br/>

* Apply suggestions from code review

avoid unnecessarily starting gramjs

Co-authored-by: Tony <TonyRL@users.noreply.github.com>

* docs: fix docs

* fix import core router

* docs: fix docs merge

* fix: eslint

* fix: guard media access with key

* docs: fix maintainer

* chore: fix pnpm @babel/*

---------

Co-authored-by: syn <syn@localhost.localdomain>
This commit is contained in:
Aleksandr Bogdanov
2024-02-29 03:00:28 +01:00
committed by GitHub
parent 645e6c0be9
commit 1badc67a6a
10 changed files with 1544 additions and 747 deletions
+4
View File
@@ -294,6 +294,10 @@ const calculateValue = () => {
},
telegram: {
token: envs.TELEGRAM_TOKEN,
session: envs.TELEGRAM_SESSION,
apiId: envs.TELEGRAM_API_ID,
apiHash: envs.TELEGRAM_API_HASH,
maxConcurrentDownloads: envs.TELEGRAM_MAX_CONCURRENT_DOWNLOADS,
},
tophub: {
cookie: envs.TOPHUB_COOKIE,
+1
View File
@@ -1,5 +1,6 @@
module.exports = {
'/blog': ['fengkx'],
'/channel/:username': ['synchrone'],
'/channel/:username/:routeParams?': ['DIYgod', 'Rongronggg9'],
'/stickerpack/:name': ['DIYgod'],
};
+14 -1
View File
@@ -1,5 +1,18 @@
const config = require('@/config').value;
const coreRouter = require('@/core-router');
module.exports = function (router) {
router.get('/channel/:username/:routeParams?', require('./channel'));
if (config.telegram.session && config.feature.mediaProxyKey) {
// core_router does not cache, which is necessary for video streaming
coreRouter.get('/telegram/channel/:username/:key/:media(.+)', require('./tglib/channel').getMedia);
}
router.get('/channel/:username/:routeParams?', (ctx) => {
// tglib impl does not support routeParams yet
const useWeb = ctx.params.routeParams || !(config.telegram.session && config.feature.mediaProxyKey);
return useWeb ? require('./channel')(ctx) : require('./tglib/channel')(ctx);
});
router.get('/stickerpack/:name', require('./stickerpack'));
router.get('/blog', require('./blog'));
};
+169
View File
@@ -0,0 +1,169 @@
const wait = require('@/utils/wait');
const config = require('@/config').value;
const { client, decodeMedia, getFilename, getMediaLink, streamDocument, streamThumbnail } = require('./client');
const bigInt = require('telegram/Helpers').returnBigInt;
const HTMLParser = require('telegram/extensions/html').HTMLParser;
function parseRange(range, length) {
if (!range) {
return [];
}
const [typ, segstr] = range.split('=');
if (typ !== 'bytes') {
throw `unsupported range: ${typ}`;
}
const segs = segstr.split(',').map((s) => s.trim());
const parsedSegs = [];
for (const seg of segs) {
const range = seg
.split('-', 2)
.filter((v) => !!v)
.map(bigInt);
if (range.length < 2) {
if (seg.startsWith('-')) {
range.unshift(0);
} else {
range.push(length);
}
}
parsedSegs.push(range);
}
return parsedSegs;
}
async function getMedia(ctx) {
if (ctx.params.key !== config.feature.mediaProxyKey) {
return ctx.throw(403, 'Invalid key');
}
const media = await decodeMedia(ctx.params.username, ctx.params.media);
if (!media) {
ctx.status = 500;
return ctx.res.end();
}
if (ctx.res.closed) {
// console.log(`prematurely closed ${ctx.params.media}`);
return;
}
if (media.document) {
ctx.status = 200;
let stream;
if ('thumb' in ctx.query) {
try {
stream = streamThumbnail(media);
ctx.set('Content-Type', 'image/jpeg');
} catch {
ctx.status = 404;
return ctx.res.end();
}
} else {
ctx.set('Content-Type', media.document.mimeType);
ctx.set('Accept-Ranges', 'bytes');
const range = parseRange(ctx.get('Range'), media.document.size - 1);
if (range.length > 1) {
ctx.status = 416; // range not satisfiable
return ctx.res.end();
}
if (range.length === 1) {
// console.log(`${ctx.method} ${ctx.req.url} Range: ${ctx.get('Range')}`);
ctx.status = 206; // partial content
const [offset, limit] = range[0];
ctx.set('Content-Length', limit - offset + 1);
ctx.set('Content-Range', `bytes ${offset}-${limit}/${media.document.size}`);
const stream = streamDocument(media.document, '', offset, limit);
for await (const chunk of stream) {
ctx.res.write(chunk);
if (ctx.res.closed) {
break;
}
}
return ctx.res.end();
}
ctx.set('Content-Length', media.document.size);
if (media.document.mimeType.startsWith('application/')) {
ctx.set('Content-Disposition', `attachment; filename="${encodeURIComponent(getFilename(media))}"`);
}
stream = streamDocument(media.document);
}
// const addr = JSON.stringify(ctx.res.socket.address());
// console.log(`streaming ${ctx.params.media} to ${addr}`);
for await (const chunk of stream) {
if (ctx.res.closed) {
// console.log(`closed ${addr}`);
break;
}
// console.log(`writing ${chunk.length / 1024} to ${addr}`);
ctx.res.write(chunk);
}
if ('close' in stream) {
stream.close();
}
} else if (media.photo) {
ctx.status = 200;
ctx.set('Content-Type', 'image/jpeg');
const buf = await client.downloadMedia(media);
ctx.res.write(buf);
} else {
ctx.status = 415;
ctx.write(media.className);
}
return ctx.res.end();
}
module.exports = async (ctx) => {
if (!config.telegram.session) {
return [];
}
if (!client.connected) {
await wait(1000);
}
const item = [];
const chat = await client.getInputEntity(ctx.params.username);
const channelInfo = await client.getEntity(chat);
let attachments = [];
const messages = await client.getMessages(chat, { limit: 50 });
for (const message of messages) {
if (message.media) {
// messages that have no text are shown as if they're one post
// because in TG only 1 attachment per message is possible
attachments.push(getMediaLink(ctx, chat, ctx.params.username, message));
}
if (message.text !== '') {
let description = attachments.join('\n');
attachments = []; // emitting these, buffer other ones
if (message.text) {
description += `<p>${HTMLParser.unparse(message.message, message.entities).replaceAll('\n', '<br/>')}</p>`;
}
const title = message.text ? message.text.substring(0, 80) + (message.text.length > 80 ? '...' : '') : new Date(message.date * 1000).toUTCString();
item.push({
title,
description,
pubDate: new Date(message.date * 1000).toUTCString(),
link: `https://t.me/s/${channelInfo.username}/${message.id}`,
author: `${channelInfo.title} (@${channelInfo.username})`,
});
}
}
ctx.state.data = {
title: channelInfo.title,
language: null,
link: `https://t.me/${channelInfo.username}`,
item,
allowEmpty: ctx.params.id === 'allow_empty',
description: `@${channelInfo.username} on Telegram`,
};
};
module.exports.getMedia = getMedia;
+194
View File
@@ -0,0 +1,194 @@
const readline = require('node:readline/promises');
const { Api, TelegramClient } = require('telegram');
const { StringSession } = require('telegram/sessions');
const { getAppropriatedPartSize } = require('telegram/Utils');
const config = require('@/config').value;
const apiId = Number(config.telegram.apiId ?? 4);
const apiHash = config.telegram.apiHash ?? '014b35b6184100b085b0d0572f9b5103';
const stringSession = new StringSession(config.telegram.session);
const client = new TelegramClient(stringSession, apiId, apiHash, {
connectionRetries: Infinity,
autoReconnect: true,
retryDelay: 3000,
maxConcurrentDownloads: Number(config.telegram.maxConcurrentDownloads ?? 10),
});
if (config.telegram.session) {
client.start({
onError: (err) => {
throw 'Cannot start TG: ' + err;
},
});
}
function humanFileSize(size) {
const i = size === 0 ? 0 : Math.floor(Math.log(size) / Math.log(1024));
return (size / Math.pow(1024, i)).toFixed(2) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i];
}
/**
* https://core.telegram.org/api/files#stripped-thumbnails
* @param bytes Buffer
* @returns Buffer jpeg
*/
function ExpandInlineBytes(bytes) {
if (bytes.length < 3 || bytes[0] !== 0x1) {
return [];
}
const header = Buffer.from([
0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, 0x4a, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0xff, 0xdb, 0x00, 0x43, 0x00, 0x28, 0x1c, 0x1e, 0x23, 0x1e, 0x19, 0x28, 0x23, 0x21, 0x23, 0x2d, 0x2b,
0x28, 0x30, 0x3c, 0x64, 0x41, 0x3c, 0x37, 0x37, 0x3c, 0x7b, 0x58, 0x5d, 0x49, 0x64, 0x91, 0x80, 0x99, 0x96, 0x8f, 0x80, 0x8c, 0x8a, 0xa0, 0xb4, 0xe6, 0xc3, 0xa0, 0xaa, 0xda, 0xad, 0x8a, 0x8c, 0xc8, 0xff, 0xcb, 0xda, 0xee,
0xf5, 0xff, 0xff, 0xff, 0x9b, 0xc1, 0xff, 0xff, 0xff, 0xfa, 0xff, 0xe6, 0xfd, 0xff, 0xf8, 0xff, 0xdb, 0x00, 0x43, 0x01, 0x2b, 0x2d, 0x2d, 0x3c, 0x35, 0x3c, 0x76, 0x41, 0x41, 0x76, 0xf8, 0xa5, 0x8c, 0xa5, 0xf8, 0xf8, 0xf8,
0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8,
0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xf8, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 0x22, 0x00, 0x02, 0x11, 0x01, 0x03, 0x11, 0x01, 0xff, 0xc4, 0x00, 0x1f, 0x00, 0x00, 0x01, 0x05,
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5, 0x10, 0x00, 0x02, 0x01, 0x03, 0x03, 0x02, 0x04,
0x03, 0x05, 0x05, 0x04, 0x04, 0x00, 0x00, 0x01, 0x7d, 0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1,
0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a,
0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96,
0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,
0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xff, 0xc4, 0x00, 0x1f, 0x01, 0x00, 0x03, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xc4, 0x00, 0xb5, 0x11, 0x00, 0x02, 0x01, 0x02, 0x04, 0x04, 0x03, 0x04, 0x07, 0x05, 0x04, 0x04, 0x00,
0x01, 0x02, 0x77, 0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62,
0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57,
0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a,
0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe2,
0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xff, 0xda, 0x00, 0x0c, 0x03, 0x01, 0x00, 0x02, 0x11, 0x03, 0x11, 0x00, 0x3f, 0x00,
]);
const footer = Buffer.from([0xff, 0xd9]);
const real = Buffer.alloc(header.length + bytes.length + footer.length);
header.copy(real);
bytes.copy(real, header.length, 3);
bytes.copy(real, 164, 1, 2);
bytes.copy(real, 166, 2, 3);
footer.copy(real, header.length + bytes.length, 0);
return real;
}
function getMediaLink(ctx, channel, channelName, message) {
const base = `${ctx.protocol}://${ctx.host}/telegram/channel/${channelName}/${config.feature.mediaProxyKey}/`;
const src = base + `${channel.channelId}_${message.id}`;
const x = message.media;
if (x instanceof Api.MessageMediaPhoto || (x instanceof Api.MessageMediaDocument && x.document.mimeType.startsWith('image/'))) {
return `<img src="${src}" alt=""/>`;
}
if (x instanceof Api.MessageMediaDocument && x.document.mimeType.startsWith('video/')) {
const vid = x.document.attributes.find((t) => t.className === 'DocumentAttributeVideo') ?? { w: 1080, h: 720 };
return `<video controls preload="metadata" poster="${src}?thumb" width="${vid.w / 2}" height="${vid.h / 2}"><source src="${src}" type="${x.document.mimeType}"></video>`;
}
if (x instanceof Api.MessageMediaDocument && x.document.mimeType.startsWith('audio/')) {
return `<audio src="${src}"></audio>`;
}
let linkText = getFilename(x);
if (x instanceof Api.MessageMediaDocument) {
linkText += ` (${humanFileSize(x.document.size)})`;
return `<a href="${src}" target="_blank"><img src="${src}?thumb" alt=""/><br/>${linkText}</a>`;
}
return;
}
function getFilename(x) {
if (x instanceof Api.MessageMediaDocument) {
const docFilename = x.document.attributes.find((a) => a.className === 'DocumentAttributeFilename');
if (docFilename) {
return docFilename.fileName;
}
}
return x.className;
}
function sortThumb(thumb) {
if (thumb instanceof Api.PhotoStrippedSize) {
return thumb.bytes.length;
}
if (thumb instanceof Api.PhotoCachedSize) {
return thumb.bytes.length;
}
if (thumb instanceof Api.PhotoSize) {
return thumb.size;
}
if (thumb instanceof Api.PhotoSizeProgressive) {
return Math.max(...thumb.sizes);
}
if (thumb instanceof Api.VideoSize) {
return thumb.size;
}
return 0;
}
function chooseLargestThumb(thumbs) {
thumbs = [...thumbs].sort((a, b) => sortThumb(a) - sortThumb(b));
return thumbs.pop();
}
function streamThumbnail(x) {
if (x instanceof Api.MessageMediaDocument && x.document.thumbs.length > 0) {
const size = chooseLargestThumb(x.document.thumbs);
if (size instanceof Api.PhotoCachedSize || size instanceof Api.PhotoStrippedSize) {
return (function* () {
yield ExpandInlineBytes(size.bytes);
})();
}
return streamDocument(x.document, size && 'type' in size ? size.type : '');
}
throw 'not supported';
}
async function decodeMedia(channelName, x, retry = false) {
const [channel, msg] = x.split('_');
try {
const msgs = await client.getMessages(channel, {
ids: [Number(msg)],
});
return msgs[0]?.media;
} catch (error) {
if (!retry) {
// channel likely not seen before, we need to resolve ID and retry
await client.getInputEntity(channelName);
return decodeMedia(channelName, x, true);
}
throw error;
}
}
function streamDocument(obj, thumbSize = '', offset, limit) {
const chunkSize = (obj.size ? getAppropriatedPartSize(obj.size) : 64) * 1024;
const iterFileParams = {
file: new Api.InputDocumentFileLocation({
id: obj.id,
accessHash: obj.accessHash,
fileReference: obj.fileReference,
thumbSize,
}),
chunkSize,
dcId: obj.dcId,
};
if (offset) {
iterFileParams.offset = offset;
}
if (limit) {
iterFileParams.limit = limit;
}
return client.iterDownload(iterFileParams);
}
module.exports = { client, getMediaLink, decodeMedia, getFilename, streamDocument, streamThumbnail };
if (require.main === module) {
Promise.resolve().then(async () => {
client.session = new StringSession('');
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
await client.start({
phoneNumber: () => rl.question('Please enter your number: '),
password: () => rl.question('Please enter your password: '),
phoneCode: () => rl.question('Please enter the code you received: '),
onError: (err) => process.stderr.write(err),
});
process.stdout.write(`TG_SESSION=${client.session.save()}\n`);
process.exit(0);
});
}
+1
View File
@@ -146,6 +146,7 @@
"simplecc-wasm": "0.1.5",
"socks-proxy-agent": "8.0.2",
"source-map": "0.7.4",
"telegram": "2.18.26",
"tiny-async-pool": "2.1.0",
"title": "3.5.3",
"tldts": "6.1.1",
+1150 -745
View File
File diff suppressed because it is too large Load Diff
+2 -1
View File
@@ -492,7 +492,8 @@ Remember to check `user-top-read` and `user-library-read` in the scope for `Pers
[Bot application](https://telegram.org/blog/bot-revolution)
- `TELEGRAM_TOKEN`: Telegram bot token
- `TELEGRAM_TOKEN`: Telegram bot token for stickerpack feeds
- `TELEGRAM_SESSION`: for video and file streaming, can be acquired by running `node lib/v2/telegram/tglib/client.js`
### Twitter
+8
View File
@@ -775,6 +775,14 @@ If the instance address is not `mastodon.social` or `pawoo.net`, then the route
:::
</Route>
### Channel with long videos and file support {#telegram-channel-with-long-videos-and-file-support}
<Route author="synchrone" example="/telegram/channel/telegram" path="/telegram/channel/:username" paramsDesc={['Channel name, without @']} configRequired="1" />
:::warning
This route requires user-based `TELEGRAM_SESSION` which can be acquired and `MEDIA_PROXY_KEY`.
:::
### Sticker Pack {#telegram-sticker-pack}
<Route author="DIYgod" example="/telegram/stickerpack/DIYgod" path="/telegram/stickerpack/:name" paramsDesc={['Sticker Pack name, available in the sharing URL']} />
@@ -472,6 +472,7 @@ RSSHub 支持使用访问密钥 / 码,允许清单和拒绝清单三种方式
贴纸包路由:[Telegram 机器人](https://telegram.org/blog/bot-revolution)
- `TELEGRAM_TOKEN`: Telegram 机器人 token
- `TELEGRAM_SESSION`: 可通过运行 `node lib/v2/telegram/tglib/client.js`
### Twitter