diff --git a/.env.example b/.env.example index 776260f4f..83cef1a97 100644 --- a/.env.example +++ b/.env.example @@ -494,6 +494,20 @@ OLLAMA_BASE_URL=http://host.docker.internal:11434 # WEKNORA_CHAT_ATTACHMENT_OCR_CONCURRENCY=8 # WEKNORA_CHAT_ATTACHMENT_OCR_MAX_PAGES=8 +# ========== E9. 飞书云文档解析模式 ========== +# 控制飞书云盘 / 飞书知识库同步新版云文档(docx)时的解析路径: +# export(默认,留空即可):走异步导出 API 下载 .docx 二进制,交给 docreader +# 解析。图片 inline 进父文档(parent_chunk_id 同 knowledge 关联),检索 / +# wiki / 智能体三个场景都能关联图片内容。代价:同步变慢;飞书 docx 内的 +# file block 附件不会随导出下载(会丢,需单独同步);若云文档中存在电子表格、 +# 多维表格等,导出后会变成内嵌在 docx 中的表格,weknora 会无法解析这些表格。 +# blocks:走 blocks API 转 Markdown。快,保留 docx 内附件;但图片渲染成空 +# `![图片]()` 占位符、并作为独立知识条目入库,与父文档无内容关联(检索 / +# wiki / 智能体均无法把图片关联回文档);电子表格会解析成 markdown 表格, +# 存在丢失合并单元格等样式风险。 +# 不需要图片关联、追求速度、要保留附件或需保留电子表格样式时设为 blocks。 +# 此方案为临时方案,若后续有更好的解析方案,会移除此环境变量并替换。 +# FEISHU_DOCX_PARSE_MODE=export # ##################################################################### # F. 认证与空间隔离 diff --git a/.gitattributes b/.gitattributes index 671cc2429..9c0d7762f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,13 @@ # acceptance/contract envelope tests. *.go text eol=lf cli/acceptance/testdata/**/*.json text eol=lf + +# Trellis: append-only developer journals should merge cleanly across +# parallel sessions/worktrees — each session only appends a new block, so +# there is nothing to actually conflict on. +# +# Do NOT add a rule for workspace/*/index.md here — it is fully regenerated +# every session, so a real conflict there is expected and safe to resolve by +# picking either side (task state lives in task.json, not index.md). See +# .trellis/spec/cli/backend/directory-structure.md for details. +.trellis/workspace/*/journal-*.md merge=union diff --git a/docker-compose.yml b/docker-compose.yml index 6bdda41eb..515503826 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -352,6 +352,8 @@ services: - OIDC_AUTH_SCOPES=${OIDC_AUTH_SCOPES:-} - OIDC_USER_INFO_MAPPING_USER_NAME=${OIDC_USER_INFO_MAPPING_USER_NAME:-} - OIDC_USER_INFO_MAPPING_EMAIL=${OIDC_USER_INFO_MAPPING_EMAIL:-} + # 飞书云文档解析模式 export:导出为docx,blocks:根据飞书云文档块解析为markdown。默认为:export + - FEISHU_DOCX_PARSE_MODE=${FEISHU_DOCX_PARSE_MODE:-export} depends_on: redis: condition: service_started diff --git a/docs/wiki/集成扩展/数据源导入开发.md b/docs/wiki/集成扩展/数据源导入开发.md index 171faedf5..cdf4e8927 100644 --- a/docs/wiki/集成扩展/数据源导入开发.md +++ b/docs/wiki/集成扩展/数据源导入开发.md @@ -29,6 +29,22 @@ WeKnora 的数据源导入模块支持从外部平台(飞书、企业微信、 > 注意:飞书国际版(Lark)同样支持,自动适配 `https://open.larksuite.com` 的 API 地址 +## 飞书 docx 解析模式与环境变量 + +飞书新版云文档(docx)的解析路径由环境变量 `FEISHU_DOCX_PARSE_MODE` 控制(作用于 app 服务,对飞书知识库和飞书云盘两个连接器同时生效): + +| 模式 | 值 | 解析路径 | 图片与文档关联 | 速度 | docx 内附件 | +|---|---|---|---|---|---| +| export(默认) | 留空 / `export` | 异步导出 -> .docx 二进制 -> docreader 解析 | ✅ 图片 inline 进父文档,`parent_chunk_id` 关联 | 慢 | 丢失 | +| blocks | `blocks` | blocks API -> Markdown | ❌ 图片作为独立知识条目,与文档割裂 | 快 | 保留 | + +**优缺点对比:** + +- **export**:导出 .docx 交 docreader 解析,图片 inline 进父文档(与普通 docx 上传一致),通过 `parent_chunk_id` 建立同知识条目父子关联,三个场景都能关联图片内容;代价是同步变慢(异步导出 + docx 解析)、docx 内附件丢失、图片 OCR/caption 依赖多模态配置。 +- **blocks**:图片 block 渲染成空 `![图片]()` 占位符,图片单独下载成独立知识条目,检索 / Wiki / 智能体无法把图片内容关联回文档;但同步快、保留 docx 内 file block 附件。 + +配置方法:默认即为 export,无需设置;需要 blocks 模式时在 `.env` 或 `docker-compose.yml` 的 app 服务环境变量中设置 `FEISHU_DOCX_PARSE_MODE=blocks`,重启 app 服务生效。详见 [飞书云盘数据源接入说明](飞书云盘数据源接入说明.md#6-docx-解析模式与环境变量)。 + ## 架构设计 ``` diff --git a/docs/wiki/集成扩展/飞书云盘数据源接入说明.md b/docs/wiki/集成扩展/飞书云盘数据源接入说明.md new file mode 100644 index 000000000..9b172a997 --- /dev/null +++ b/docs/wiki/集成扩展/飞书云盘数据源接入说明.md @@ -0,0 +1,167 @@ +# 飞书云盘数据源使用说明 + +飞书云盘数据源(`feishu_drive` / `lark_drive`)可以把飞书/Lark 云盘某个文件夹下的文档和文件自动同步到 WeKnora 知识库,支持增量同步、定时同步和子文件夹递归。 + +--- + +## 1. 前置条件:创建飞书应用 + +1. 登录[飞书开放平台](https://open.feishu.cn/app)(Lark 用户使用 [Lark 开放平台](https://open.larksuite.com/app)),创建**企业自建应用**。 +2. 记录应用的 **App ID**(`cli_` 开头)和 **App Secret**,配置数据源时需要填写。 +3. 按下节表格开通权限,并**发布应用版本**(权限修改后必须重新发布才生效)。 + +> 注意:飞书(open.feishu.cn)和 Lark(open.larksuite.com)是两个独立体系,应用不通用。同步飞书云盘用飞书应用,同步 Lark Drive 用 Lark 应用,凭据不能混用。 + +--- + +## 2. 所需权限(详细) + +在应用后台「权限管理」中开通以下 **3 个权限**: + +| 权限标识 | 名称 | 用途 | 缺少时的表现 | +|---|---|---|---| +| `drive:drive:readonly` | 查看云空间中的文件 | 列举文件夹内容(list API)、下载云盘普通文件 | 加载文件夹/同步时报 403,提示「需先将文件夹分享给应用所在的群」 | +| `drive:export:readonly` | 导出云文档 | 把 docx/doc/sheet/bitable 导出为 docx/xlsx 再解析 | 云文档类文件同步失败 | +| `docx:document:readonly` | 读取新版文档内容 | 通过 blocks API 解析 docx 文档正文与附件(导出失败时的主路径) | docx 文档解析失败或回退导出也失败 | + +说明: + +- 与「飞书知识库」连接器相比,云盘**不需要** `wiki:wiki:readonly`,其余权限相同。 +- list API 本身还接受 `drive:drive`(读写)或 `space:document:retrieve` 作为替代,但推荐只开只读的 `drive:drive:readonly`,最小授权。 +- 权限开通后必须**创建并发布新版本**,否则接口仍报无权限。 + +--- + +## 3. 关键一步:把文件夹分享给应用 + +飞书的权限模型要求:即使应用开通了上述 API 权限,也只能访问**被显式分享给它的文件**。 + +1. 创建一个飞书(Lark)群或复用某个飞书(Lark)群 +2. 将需要导入的**飞书云盘管理者**拉进群 +> PS: 不拉进群也会导致没有访问权限 +3. 在飞书云盘中打开目标文件夹; +4. 点击「分享」/「··· → 添加协作者」,把文件夹分享给**应用所在的群**(把应用拉进一个群,再将文件夹分享给该群),权限给「可阅读」即可; +5. 子文件夹和其中的文件会随父文件夹一起获得授权,无需逐个分享; +6. 如果之后新增的文件同步报 403,检查该文件是否在这棵已分享的目录树下。 + +这是一次性操作,但不做的话,加载文件夹会直接报「应用无权访问该文件夹」。 + +--- + +## 4. 配置数据源(四步) + +入口:知识库 → 设置 → 数据源 → 新建数据源,选择「飞书云盘」。 + +### 第 1 步:选择类型 + +选择「飞书云盘」(国际版选「Lark Drive」)。 + +### 第 2 步:配置凭证 + +填写 App ID 和 App Secret,点击下一步时系统会自动测试连接(验证 tenant_access_token 能否获取)。此步只验证应用身份,不验证文件夹权限。 + +### 第 3 步:选择范围 + +1. 在「云盘文件夹 Token」输入框填入目标文件夹的 `folder_token`,**或直接粘贴文件夹的完整链接**(飞书 `https://xxx.feishu.cn/drive/folder/` 或 Lark `https://xxx.larksuite.com/drive/folder/`,系统按路径自动提取 token,两种链接都支持); +2. 点击「加载」,列出该文件夹下的完整目录树; +3. 勾选要同步的文件/文件夹,支持逐级展开、全选/折叠分支。 + +注意: + +- **不支持云空间根目录**(根目录不分页且不返回快捷方式),必须选择具体文件夹; +- 加载失败时按提示处理:403 → 回到第 3 节分享文件夹;token 无效 → 重新从文件夹 URL 复制。 + +### 第 4 步:同步策略 + +| 配置项 | 说明 | 默认值 | +|---|---|---| +| 同步计划 | cron 表达式,默认每 6 小时一次;留空则只手动触发 | `0 0 */6 * * *` | +| 同步模式 | 增量(按修改时间游标)/ 全量 | 增量 | +| 冲突策略 | 内容变更时覆盖 / 跳过 | 覆盖 | +| 同步删除 | 源端删除的文档**只计数,不自动删除知识库内容**,需在知识库手动删除 | 开启 | + +保存后数据源开始按策略运行,也可在数据源卡片上手动「触发同步」。 + +--- + +## 5. 支持的文件类型 + +| 云盘类型 | 处理方式 | +|---|---| +| `docx` / `doc`(新旧文档) | blocks API 解析正文与附件,失败时回退导出为 docx 解析 | +| `sheet` / `bitable`(表格/多维表格) | 导出为 xlsx 解析 | +| `file`(普通上传文件,如 PDF/PPT/图片) | 直接下载后按文件类型解析 | +| `shortcut`(快捷方式) | 自动解析为目标文件同步(快捷方式不能指向文件夹) | +| `folder`(文件夹) | 递归遍历 | +| `mindnote` / `slides` / `board` | 不支持,跳过 | + +补充行为: + +- docx 中的**附件**会作为独立知识条目同步(与父文档关联,父文档更新时自动清理已移除的附件); +- 文档内嵌图片会尝试 OCR/多模态解析,未配置对象存储或 VLM 时自动跳过,不影响正文同步。 + +--- + +## 6. docx 解析模式与环境变量 + +飞书新版云文档(docx)有两种解析路径,由环境变量 `FEISHU_DOCX_PARSE_MODE` 控制。该变量作用于 WeKnora **app 服务**(不是数据源配置),对飞书云盘和飞书知识库两个连接器同时生效。 + +### 模式对比 + +| | export(默认) | blocks | +|---|---|---| +| 环境变量值 | 留空 / `export` | `blocks` | +| 解析路径 | 异步导出 API -> .docx 二进制 -> docreader 解析 | blocks API -> Markdown | +| 图片与文档关联 | ✅ 图片 inline 进父文档,`parent_chunk_id` 关联 | ❌ 图片作为独立知识条目,与文档割裂 | +| 检索 / Wiki / 智能体能否关联图片 | 是 | 否 | +| 同步速度 | 慢(异步导出 + docx 解析) | 快 | +| docx 内附件(file block) | 丢失(.docx 导出不含) | 保留,作为独立条目 | +| 所需权限 | `drive:drive:readonly` + `drive:export:readonly` | `drive:drive:readonly` + `drive:export:readonly` + `docx:document:readonly` | + +### 为什么图片关联有差异 + +- **export 模式**:导出 .docx 后由 docreader 解析,图片 inline 进父文档(与普通 docx 上传一致),通过 `parent_chunk_id` 建立同知识条目的父子关联,三个场景都能在一次检索中把图片内容与文档一起返回。 +- **blocks 模式**:走 blocks API,图片 block 渲染成空 `![图片]()` 占位符,图片单独下载成独立知识条目,与父文档只有元数据级弱关联。WeKnora 的检索、Wiki 构建、智能体问答链路都不会把图片内容关联回文档,图片和正文是割裂的。 + +### 配置方法 + +在 WeKnora 服务的 `.env` 或 `docker-compose.yml` 的 app 服务环境变量中设置: + +```env +FEISHU_DOCX_PARSE_MODE=blocks +``` + +不设置或设为 `export` 即用默认模式。修改后需重启 app 服务生效。 + +### export 模式的代价 + +- **同步变慢**:每个 docx 都要走异步导出(创建任务 + 轮询 + 下载)+ docreader 解析,比 blocks API 慢。 +- **附件丢失**:docx 内 file block 附件不随 .docx 导出下载,如需附件用 blocks 模式或单独同步。 +- **图片内容依赖多模态**:图片 inline 后 OCR/caption 由多模态服务异步生成,未配置对象存储或 VLM 时图片只存储不生成内容(前端展示正常,但检索层面仍弱)。 + +### 选择建议 + +- 需要图片内容在检索 / Wiki / 智能体中与文档关联:用默认 **export**。 +- 只需文档正文、要保留附件、追求同步速度:用 **blocks**。 + +--- + +## 7. 同步行为说明 + +- **增量同步**:以文件修改时间为游标,只拉取上次同步后变更的内容;中断后从断点续传。 +- **部分失败不中断**:某个子文件夹无权限或某个文件下载失败时,该条目记为失败,其余内容继续同步,失败明细可在「同步日志」中查看。 +- **更新语义**:内容变更的文件会先删除旧知识条目再重建,解析期间该文档短暂不可用,属正常现象。 +- **安全约束**:为避免误删,源端删除的文件不会自动从知识库移除(见第 4 步「同步删除」)。 + +--- + +## 8. 常见问题 + +| 现象 | 原因与处理 | +|---|---| +| 「请输入具体文件夹的 folder_token,不支持云空间根目录」 | 输入为空或粘贴的是根目录链接,换具体文件夹链接 | +| 「应用无权访问该文件夹。请…分享给应用所在的群」 | 未完成第 3 节的分享,或分享的对象不是应用所在的群 | +| 「应用凭证无效或缺少云盘权限」 | App ID/Secret 错误,或第 2 节权限未开通/未发布版本 | +| 「folder_token 不存在或已删除」 | token 复制有误,从文件夹「分享 → 复制链接」重新获取 | +| 同步日志中部分条目失败 | 点开日志看失败阶段:`list_children` 多为子文件夹未授权,`fetch` 多为单文件权限或类型不支持 | +| 知识列表中来源显示 | 云盘同步的文档来源标记为「飞书云盘」,与知识库同步的「飞书」区分 | diff --git a/docs/数据源导入开发文档.md b/docs/数据源导入开发文档.md index 875880e0f..cfb6cd444 100644 --- a/docs/数据源导入开发文档.md +++ b/docs/数据源导入开发文档.md @@ -695,7 +695,7 @@ GET /open-apis/wiki/v2/spaces (分页, page_size=50) | `obj_type` | 支持 | 获取方式 | 导出格式 | |------------|------|---------|---------| -| `docx` | 是 | 导出任务 (Export API) | `.docx` | +| `docx` | 是 | 导出 .docx(默认)/ blocks API 转 Markdown(blocks 模式) | `.docx` / Markdown | | `doc` | 是 | 导出任务 (Export API) | `.docx` | | `sheet` | 是 | 导出任务 (Export API) | `.xlsx` | | `bitable` | 是 | 导出任务 (Export API) | `.xlsx` | @@ -705,29 +705,63 @@ GET /open-apis/wiki/v2/spaces (分页, page_size=50) #### 内容获取流程 -**文档 (docx/doc/sheet/bitable):** +**docx(新版云文档):** 由环境变量 `FEISHU_DOCX_PARSE_MODE` 控制解析路径(作用于 app 服务,飞书知识库和飞书云盘连接器同时生效)。 -``` -1. CreateExportTask → 创建导出任务 -2. 轮询 GetExportTaskStatus(间隔 2 秒,最长约 60 秒) -3. DownloadExportFile → 用 file_token 下载导出文件 -4. 清理文件名 (sanitizeFileName) + 补全扩展名 -``` +- **export 模式(默认,留空或 `export`)**:走异步导出 API + ``` + 1. CreateExportTask -> 创建导出任务 + 2. 轮询 GetExportTaskStatus(间隔 2 秒,最长约 60 秒) + 3. DownloadExportFile -> 下载 .docx 二进制 + 4. .docx 交 docreader 解析(图片 inline 进父文档,与普通 docx 上传一致) + ``` +- **blocks 模式(`FEISHU_DOCX_PARSE_MODE=blocks`)**:走 blocks API + ``` + 1. GET /open-apis/docx/v1/documents/{obj_token}/blocks (分页 500) + 2. blocksToMarkdown -> 转换为 Markdown 正文 + ├─ 文本/标题/列表/表格/代码块 -> Markdown + ├─ image block -> ![图片]() 空占位符(图片单独下载为独立知识条目) + └─ file block -> 附件,作为独立知识条目 + 3. blocks API 失败或渲染空 -> 回退 export + ``` + +**doc / sheet / bitable:** 走异步导出 API(同 export 模式流程),导出为 .docx / .xlsx 后交 docreader 解析。 **文件 (file):** ``` -DownloadDriveFile → GET /drive/v1/files/{token}/download +DownloadDriveFile -> GET /drive/v1/files/{token}/download ``` +##### blocks vs export 优缺点 + +| | export(默认) | blocks | +|---|---|---| +| 解析路径 | 导出 .docx -> docreader | blocks API -> Markdown | +| 图片与文档关联 | ✅ 图片 inline 进父文档,`parent_chunk_id` 关联 | ❌ 图片作为独立知识条目,与文档割裂 | +| 检索 / Wiki / 智能体关联图片 | 是 | 否 | +| 同步速度 | 慢(异步导出 + docx 解析) | 快 | +| docx 内附件(file block) | 丢失(.docx 导出不含) | 保留 | +| 所需权限 | 不需要(但建议保留以便切换) | 需 `docx:document:readonly` | + +- **export**:图片 inline 进父文档(同普通 docx 上传),通过 `parent_chunk_id` 建立同知识条目父子关联,三个场景都能关联图片内容;代价是同步慢、docx 内附件丢失、图片 OCR/caption 依赖多模态配置。 +- **blocks**:快、保留附件,但图片 block 渲染成空占位符、图片独立入库,与父文档只有元数据级弱关联,检索 / Wiki / 智能体都无法把图片内容关联回文档。 + +> 环境变量配置见 `.env.example` 的 E9 段。 + #### 源码文件 | 文件 | 职责 | |------|------| -| `internal/datasource/connector/feishu/types.go` | 飞书 API 类型定义、配置结构、常量 | -| `internal/datasource/connector/feishu/client.go` | API 客户端:Token 管理、Wiki/Drive API 调用、导出/下载 | -| `internal/datasource/connector/feishu/connector.go` | Connector 接口实现:Validate、ListResources、FetchAll、FetchIncremental | -| `internal/datasource/connector/feishu/connector_test.go` | 单元测试:使用 HTTP Mock 模拟飞书开放平台 | +| `internal/datasource/connector/feishu/core/types.go` | 飞书 API 类型定义、配置结构(Config)、Region 常量 | +| `internal/datasource/connector/feishu/core/client.go` | API 客户端:Token 管理、Wiki/Drive API 调用、导出/下载 | +| `internal/datasource/connector/feishu/core/blocks.go` | docx blocks API 类型(DocxBlock 等)、listDocumentBlocks | +| `internal/datasource/connector/feishu/core/markdown.go` | blocksToMarkdown:block 数组转 Markdown | +| `internal/datasource/connector/feishu/core/shared.go` | 共享逻辑:FetchDocxWithBlocks、ParseFeishuConfig、exportDocxFallback | +| `internal/datasource/connector/feishu/core/engine.go` | 通用同步引擎:NodeOps 接口、FetchStreamEngine / FetchAllEngine | +| `internal/datasource/connector/feishu/core/region.go` | Region(飞书 / Lark 云区分)、URL 构造 | +| `internal/datasource/connector/feishu/wiki/connector.go` | 飞书知识库 Connector 实现(wikiOps) | +| `internal/datasource/connector/feishu/drive/connector.go` | 飞书云盘 Connector 实现(driveOps) | +| `internal/datasource/connector/feishu/{core,wiki,drive}/*_test.go` | 单元测试:使用 HTTP Mock 模拟飞书开放平台 | ## 定时调度 diff --git a/frontend/src/components/doc-content.vue b/frontend/src/components/doc-content.vue index 0b2652dc2..0e04a2db0 100644 --- a/frontend/src/components/doc-content.vue +++ b/frontend/src/components/doc-content.vue @@ -967,6 +967,10 @@ const channelLabelMap: Record = { wechat: 'knowledgeBase.channelWechat', wecom: 'knowledgeBase.channelWecom', feishu: 'knowledgeBase.channelFeishu', + // Drive (云盘) connectors get their own channel so Drive docs show + // "飞书云盘" / "Lark 云盘", distinct from the wiki connector's "飞书". + feishu_drive: 'knowledgeBase.channelFeishuDrive', + lark_drive: 'knowledgeBase.channelLarkDrive', dingtalk: 'knowledgeBase.channelDingtalk', slack: 'knowledgeBase.channelSlack', im: 'knowledgeBase.channelIm', diff --git a/frontend/src/i18n/locales/en-US.ts b/frontend/src/i18n/locales/en-US.ts index 472ff34dd..d28822f68 100755 --- a/frontend/src/i18n/locales/en-US.ts +++ b/frontend/src/i18n/locales/en-US.ts @@ -459,6 +459,8 @@ export default { channelWechat: 'WeChat', channelWecom: 'WeCom', channelFeishu: 'Feishu', + channelFeishuDrive: 'Feishu Drive', + channelLarkDrive: 'Lark Drive', channelDingtalk: 'DingTalk', channelSlack: 'Slack', channelIm: 'IM Channel', @@ -5252,6 +5254,8 @@ export default { connector: { feishu: 'Feishu', lark: 'Lark', + feishu_drive: 'Feishu Drive', + lark_drive: 'Lark Drive', notion: 'Notion', yuque: 'Yuque', rss: 'RSS / Atom Feed' @@ -5259,17 +5263,32 @@ export default { connectorDesc: { feishu: 'Sync documents, spreadsheets and files from Feishu Wiki', lark: 'Sync documents, spreadsheets and files from Lark Wiki (Feishu international)', + feishu_drive: 'Sync documents, spreadsheets and files from a Feishu Drive folder', + lark_drive: 'Sync documents, spreadsheets and files from a Lark Drive folder (Feishu international)', notion: 'Sync pages and databases from Notion', yuque: 'Sync documents from Yuque knowledge bases', rss: 'Sync articles from RSS / Atom feeds' }, + drive: { + folderTokenLabel: 'Drive folder token', + folderTokenPlaceholder: 'Enter a folder_token or a Feishu Drive folder URL', + folderTokenRequired: 'Please enter a concrete folder_token; the cloud-space root is not supported', + rootNotSupportedHint: 'The root folder is not paginated and does not return shortcuts; pick a concrete folder', + load: 'Load', + shareHint: 'Share the Drive folder with the app’s group first, otherwise the app cannot access it', + placeholderTitle: 'Load a Drive folder first', + placeholderDesc: 'Enter a folder_token (or paste a Feishu Drive folder URL) above and click "Load"', + loadForbiddenHint: 'The app has no access to this folder. Share the folder with the app’s group in Feishu Drive and retry.', + loadAuthHint: 'App credentials are invalid or missing Drive scopes. Check App ID / App Secret and drive:drive:readonly permissions.', + loadNotFoundHint: 'folder_token does not exist or has been deleted. Verify the token copied from the Feishu Drive folder URL.', + }, field: { appId: 'App ID', appSecret: 'App Secret', integrationToken: 'Integration Token', apiToken: 'API Token', baseUrl: 'Base URL (optional)', - baseUrlHint: 'Leave empty to use the Yuque public cloud (https://www.yuque.com). For Yuque Enterprise or self-hosted deployments, enter your company domain (e.g. https://your-company.yuque.com).', + baseUrlHint: 'Leave empty to use the default public cloud address. For private/enterprise deployments or when accessing via reverse proxy, enter your custom address (e.g. https://api-proxy.example.com).', feedUrls: 'Feed URLs', feedUrlsHint: 'One RSS / Atom feed URL per line; multiple feeds are supported.', authHeaders: 'Custom headers (optional)', @@ -5286,6 +5305,30 @@ export default { prereqStep2Desc_yuque: 'Check at least repo:read and doc:read (read knowledge base and document content)', prereqStep3Brief_yuque: '(Optional) Enter Base URL for enterprise deployments', prereqStep3Desc_yuque: 'Leave empty for public cloud; for Yuque Enterprise or self-hosted, enter your company domain.', + prereqStep1Brief_feishu: 'Create Feishu custom app', + prereqStep1Desc_feishu: 'Login Feishu Open Platform → Create enterprise custom app', + prereqStep2Brief_feishu: 'Add bot capability', + prereqStep2Desc_feishu: 'Open Platform → Your app → Add app capability → Bot', + prereqStep3Brief_feishu: 'Configure app permissions', + prereqStep3Desc_feishu: 'Enable wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly permissions', + prereqStep1Brief_lark: 'Create Lark custom app', + prereqStep1Desc_lark: 'Login Lark Open Platform → Create enterprise custom app', + prereqStep2Brief_lark: 'Add bot capability', + prereqStep2Desc_lark: 'Open Platform → Your app → Add app capability → Bot', + prereqStep3Brief_lark: 'Configure app permissions', + prereqStep3Desc_lark: 'Enable wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly permissions', + prereqStep1Brief_feishu_drive: 'Create Feishu custom app', + prereqStep1Desc_feishu_drive: 'Login Feishu Open Platform → Create enterprise custom app', + prereqStep2Brief_feishu_drive: 'Add bot capability', + prereqStep2Desc_feishu_drive: 'Open Platform → Your app → Add app capability → Bot', + prereqStep3Brief_feishu_drive: 'Configure app permissions', + prereqStep3Desc_feishu_drive: 'Enable drive:drive:readonly, drive:export:readonly, docx:document:readonly permissions', + prereqStep1Brief_lark_drive: 'Create Lark custom app', + prereqStep1Desc_lark_drive: 'Login Lark Open Platform → Create enterprise custom app', + prereqStep2Brief_lark_drive: 'Add bot capability', + prereqStep2Desc_lark_drive: 'Open Platform → Your app → Add app capability → Bot', + prereqStep3Brief_lark_drive: 'Configure app permissions', + prereqStep3Desc_lark_drive: 'Enable drive:drive:readonly, drive:export:readonly, docx:document:readonly permissions', prereqOpenConsole_yuque: 'Open Yuque Token settings', prereqBotBrief: 'Add "Bot" capability to your app', prereqBotDesc: 'Open Platform > Add App Capability > Bot > create version and publish', diff --git a/frontend/src/i18n/locales/ko-KR.ts b/frontend/src/i18n/locales/ko-KR.ts index dc7dc689f..4f7b4cfef 100755 --- a/frontend/src/i18n/locales/ko-KR.ts +++ b/frontend/src/i18n/locales/ko-KR.ts @@ -595,6 +595,30 @@ export default { prereqStep2Desc_yuque: '최소한 repo:read 와 doc:read 를 선택하세요 (지식베이스 및 문서 콘텐츠 읽기)', prereqStep3Brief_yuque: '(선택) Enterprise 사용 시 Base URL 입력', prereqStep3Desc_yuque: '퍼블릭 클라우드 사용자는 입력하지 않아도 됩니다. Yuque Enterprise 또는 사설 배포 시 기업 도메인을 입력하세요', + prereqStep1Brief_feishu: "Feishu 커스텀 앱 생성", + prereqStep1Desc_feishu: "Feishu Open Platform 로그인 → 엔터프라이즈 커스텀 앱 생성", + prereqStep2Brief_feishu: "봇 기능 추가", + prereqStep2Desc_feishu: "Open Platform → 앱 → 앱 기능 추가 → 봇", + prereqStep3Brief_feishu: "앱 권한 구성", + prereqStep3Desc_feishu: "wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly 권한 활성화", + prereqStep1Brief_lark: "Lark 커스텀 앱 생성", + prereqStep1Desc_lark: "Lark Open Platform 로그인 → 엔터프라이즈 커스텀 앱 생성", + prereqStep2Brief_lark: "봇 기능 추가", + prereqStep2Desc_lark: "Open Platform → 앱 → 앱 기능 추가 → 봇", + prereqStep3Brief_lark: "앱 권한 구성", + prereqStep3Desc_lark: "wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly 권한 활성화", + prereqStep1Brief_feishu_drive: "Feishu 커스텀 앱 생성", + prereqStep1Desc_feishu_drive: "Feishu Open Platform 로그인 → 엔터프라이즈 커스텀 앱 생성", + prereqStep2Brief_feishu_drive: "봇 기능 추가", + prereqStep2Desc_feishu_drive: "Open Platform → 앱 → 앱 기능 추가 → 봇", + prereqStep3Brief_feishu_drive: "앱 권한 구성", + prereqStep3Desc_feishu_drive: "drive:drive:readonly, drive:export:readonly, docx:document:readonly 권한 활성화", + prereqStep1Brief_lark_drive: "Lark 커스텀 앱 생성", + prereqStep1Desc_lark_drive: "Lark Open Platform 로그인 → 엔터프라이즈 커스텀 앱 생성", + prereqStep2Brief_lark_drive: "봇 기능 추가", + prereqStep2Desc_lark_drive: "Open Platform → 앱 → 앱 기능 추가 → 봇", + prereqStep3Brief_lark_drive: "앱 권한 구성", + prereqStep3Desc_lark_drive: "drive:drive:readonly, drive:export:readonly, docx:document:readonly 권한 활성화", prereqOpenConsole_yuque: 'Yuque Token 설정으로 이동', prereqBotBrief: '앱에 \'봇\' 기능 추가', prereqBotDesc: '오픈 플랫폼 → 앱 기능 추가 → 봇 → 버전 생성 후 게시', @@ -633,7 +657,7 @@ export default { integrationToken: 'Integration Token', apiToken: 'API Token', baseUrl: 'Base URL', - baseUrlHint: '비워두면 Yuque 퍼블릭 클라우드 https://www.yuque.com 를 사용합니다. Yuque Enterprise 또는 사설 배포를 사용하는 경우 기업 도메인(예: https://your-company.yuque.com)을 입력하세요', + baseUrlHint: "비워두면 기본 퍼블릭 클라우드 주소가 사용됩니다. 프라이빗/엔터프라이즈 배포거나 리버스 프록시를 통해 액세스해야 하는 경우 사용자 정의 주소를 입력하세요 (예: https://api-proxy.example.com)", feedUrls: '피드 주소', feedUrlsHint: '한 줄에 하나씩 RSS / Atom 피드 주소를 입력하세요. 여러 개를 함께 입력할 수 있습니다.', authHeaders: '사용자 지정 헤더 (선택)', @@ -642,6 +666,8 @@ export default { connectorDesc: { feishu: '페이슈 위키에서 문서, 스프레드시트, 파일 동기화', lark: 'Lark 위키에서 문서, 스프레드시트, 파일 동기화', + feishu_drive: "페이슈 드라이브 폴더에서 문서, 스프레드시트, 파일 동기화", + lark_drive: "Lark 드라이브 폴더에서 문서, 스프레드시트, 파일 동기화", notion: 'Notion에서 페이지 및 데이터베이스 동기화', yuque: '위큐 지식베이스에서 문서 동기화', rss: 'RSS / Atom 피드에서 글 동기화' @@ -649,6 +675,8 @@ export default { connector: { feishu: '페이슈 (Feishu)', lark: 'Lark (Feishu 글로벌)', + feishu_drive: "페이슈 드라이브", + lark_drive: "Lark 드라이브", notion: 'Notion', yuque: '위큐 (Yuque)', rss: 'RSS / Atom 피드' @@ -696,7 +724,20 @@ export default { syncMode: { incremental: '증분 동기화', full: '전체 동기화' - } + }, + drive: { + folderTokenLabel: "드라이브 폴더 토큰", + folderTokenPlaceholder: "folder_token 또는 페이슈 드라이브 폴더 URL 입력", + folderTokenRequired: "구체적인 폴더의 folder_token을 입력하세요. 클라우드 루트는 지원되지 않습니다", + rootNotSupportedHint: "루트 폴더는 페이지네이션되지 않고 바로가기를 반환하지 않습니다. 구체적인 폴더를 선택하세요", + load: "로드", + shareHint: "앱이 접근할 수 있도록 먼저 드라이브 폴더를 앱이 속한 그룹에 공유하세요", + placeholderTitle: "먼저 드라이브 폴더를 로드하세요", + placeholderDesc: "위에 folder_token(또는 페이슈 드라이브 폴더 URL)을 입력하고 '로드'를 클릭하세요", + loadForbiddenHint: "앱이 이 폴더에 접근할 수 없습니다. 페이슈 드라이브에서 폴더를 앱이 속한 그룹에 공유한 후 다시 시도하세요.", + loadAuthHint: "앱 자격 증명이 유효하지 않거나 드라이브 권한이 없습니다. App ID / App Secret 및 drive:drive:readonly 권한을 확인하세요.", + loadNotFoundHint: "folder_token이 존재하지 않거나 삭제되었습니다. 페이슈 드라이브 폴더 URL에서 복사한 토큰이 맞는지 확인하세요.", + }, }, ollama: { unknown: '알 수 없음', @@ -5299,6 +5340,8 @@ export default { channelWechat: 'WeChat', channelWecom: 'WeCom', channelFeishu: 'Feishu', + channelFeishuDrive: "페이슈 드라이브", + channelLarkDrive: "Lark 드라이브", channelDingtalk: 'DingTalk', channelSlack: 'Slack', channelIm: 'IM 채널', diff --git a/frontend/src/i18n/locales/ru-RU.ts b/frontend/src/i18n/locales/ru-RU.ts index 9dbc48cda..dbb4af1ac 100755 --- a/frontend/src/i18n/locales/ru-RU.ts +++ b/frontend/src/i18n/locales/ru-RU.ts @@ -596,6 +596,30 @@ export default { prereqStep3Brief_yuque: '(Опционально) Для Enterprise укажите Base URL', prereqStep3Desc_yuque: 'Пользователям публичного облака указывать не нужно. Для Yuque Enterprise или приватного развёртывания укажите корпоративный домен', prereqOpenConsole_yuque: 'Перейти к настройкам Yuque Token', + prereqStep1Brief_feishu: 'Создать частное приложение Feishu', + prereqStep1Desc_feishu: 'Войдите в Feishu Open Platform → Создать корпоративное частное приложение', + prereqStep2Brief_feishu: 'Добавить возможность бота', + prereqStep2Desc_feishu: 'Open Platform → Ваше приложение → Добавить возможность приложения → Bot', + prereqStep3Brief_feishu: 'Настроить разрешения приложения', + prereqStep3Desc_feishu: 'Включите разрешения: wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly', + prereqStep1Brief_lark: 'Создать частное приложение Lark', + prereqStep1Desc_lark: 'Войдите в Lark Open Platform → Создать корпоративное частное приложение', + prereqStep2Brief_lark: 'Добавить возможность бота', + prereqStep2Desc_lark: 'Open Platform → Ваше приложение → Добавить возможность приложения → Bot', + prereqStep3Brief_lark: 'Настроить разрешения приложения', + prereqStep3Desc_lark: 'Включите разрешения: wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly', + prereqStep1Brief_feishu_drive: 'Создать частное приложение Feishu', + prereqStep1Desc_feishu_drive: 'Войдите в Feishu Open Platform → Создать корпоративное частное приложение', + prereqStep2Brief_feishu_drive: 'Добавить возможность бота', + prereqStep2Desc_feishu_drive: 'Open Platform → Ваше приложение → Добавить возможность приложения → Bot', + prereqStep3Brief_feishu_drive: 'Настроить разрешения приложения', + prereqStep3Desc_feishu_drive: 'Включите разрешения: drive:drive:readonly, drive:export:readonly, docx:document:readonly', + prereqStep1Brief_lark_drive: 'Создать частное приложение Lark', + prereqStep1Desc_lark_drive: 'Войдите в Lark Open Platform → Создать корпоративное частное приложение', + prereqStep2Brief_lark_drive: 'Добавить возможность бота', + prereqStep2Desc_lark_drive: 'Open Platform → Ваше приложение → Добавить возможность приложения → Bot', + prereqStep3Brief_lark_drive: 'Настроить разрешения приложения', + prereqStep3Desc_lark_drive: 'Включите разрешения: drive:drive:readonly, drive:export:readonly, docx:document:readonly', prereqBotBrief: 'Добавьте приложению возможность «Бот»', prereqBotDesc: 'Открытая платформа → Добавить возможность приложения → Бот → Создать версию и опубликовать', prereqPermBrief: 'Включите права API', @@ -633,7 +657,7 @@ export default { integrationToken: 'Integration Token', apiToken: 'API Token', baseUrl: 'Base URL', - baseUrlHint: 'Оставьте пустым, чтобы использовать публичное облако Yuque https://www.yuque.com. Если вы используете Yuque Enterprise или приватное развёртывание, укажите корпоративный домен (например, https://your-company.yuque.com)', + baseUrlHint: 'Оставьте пустым, чтобы использовать адрес общедоступного облака по умолчанию. Для частных/корпоративных развертываний или при доступе через обратный прокси введите ваш собственный адрес (например, https://api-proxy.example.com)', feedUrls: 'Адреса лент', feedUrlsHint: 'По одному адресу ленты RSS / Atom в строке; можно указать несколько.', authHeaders: 'Пользовательские заголовки (необязательно)', @@ -642,6 +666,8 @@ export default { connectorDesc: { feishu: 'Синхронизация документов, таблиц и файлов из Feishu Wiki', lark: 'Синхронизация документов, таблиц и файлов из Lark Wiki', + feishu_drive: 'Синхронизация документов, таблиц и файлов из папки Feishu Drive', + lark_drive: 'Синхронизация документов, таблиц и файлов из папки Lark Drive', notion: 'Синхронизация страниц и баз данных из Notion', yuque: 'Синхронизация документов из баз знаний Yuque', rss: 'Синхронизация статей из лент RSS / Atom' @@ -649,6 +675,8 @@ export default { connector: { feishu: 'Feishu (Фэйшу)', lark: 'Lark', + feishu_drive: 'Feishu Drive', + lark_drive: 'Lark Drive', notion: 'Notion', yuque: 'Yuque (Юйцюэ)', rss: 'RSS / Atom лента' @@ -696,7 +724,20 @@ export default { syncMode: { incremental: 'Инкрементная', full: 'Полная' - } + }, + drive: { + folderTokenLabel: 'Токен папки Drive', + folderTokenPlaceholder: 'Введите folder_token или URL папки Feishu Drive', + folderTokenRequired: 'Введите конкретный folder_token; корень облачного пространства не поддерживается', + rootNotSupportedHint: 'Корневая папка не поддерживает постраничный вывод и не возвращает ярлыки; выберите конкретную папку', + load: 'Загрузить', + shareHint: 'Сначала поделитесь папкой Drive с группой приложения, иначе приложение не получит к ней доступ', + placeholderTitle: 'Сначала загрузите папку Drive', + placeholderDesc: 'Введите выше folder_token (или вставьте URL папки Feishu Drive) и нажмите «Загрузить»', + loadForbiddenHint: 'У приложения нет доступа к этой папке. Поделитесь папкой с группой приложения в Feishu Drive и повторите.', + loadAuthHint: 'Учётные данные приложения недействительны или отсутствуют области Drive. Проверьте App ID / App Secret и разрешения drive:drive:readonly.', + loadNotFoundHint: 'folder_token не существует или удалён. Проверьте токен, скопированный из URL папки Feishu Drive.', + }, }, ollama: { unknown: 'Неизвестно', @@ -5299,6 +5340,8 @@ export default { channelWechat: 'WeChat', channelWecom: 'WeCom', channelFeishu: 'Feishu', + channelFeishuDrive: 'Feishu Drive', + channelLarkDrive: 'Lark Drive', channelDingtalk: 'DingTalk', channelSlack: 'Slack', channelIm: 'IM канал', diff --git a/frontend/src/i18n/locales/zh-CN.ts b/frontend/src/i18n/locales/zh-CN.ts index cd9ea1507..06eb7a75c 100755 --- a/frontend/src/i18n/locales/zh-CN.ts +++ b/frontend/src/i18n/locales/zh-CN.ts @@ -596,6 +596,30 @@ export default { prereqStep3Brief_yuque: '(可选)企业版填写 Base URL', prereqStep3Desc_yuque: '公有云用户无需填写;语雀企业版或私有部署请填写企业域名', prereqOpenConsole_yuque: '前往语雀 Token 设置', + prereqStep1Brief_feishu: "创建飞书自建应用", + prereqStep1Desc_feishu: "登录飞书开放平台 → 创建企业自建应用", + prereqStep2Brief_feishu: "添加机器人能力", + prereqStep2Desc_feishu: "开放平台 → 你的应用 → 添加应用能力 → 机器人", + prereqStep3Brief_feishu: "配置应用权限", + prereqStep3Desc_feishu: "为应用开通 wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly 权限", + prereqStep1Brief_lark: "创建 Lark 自建应用", + prereqStep1Desc_lark: "登录 Lark 开放平台 → 创建企业自建应用", + prereqStep2Brief_lark: "添加机器人能力", + prereqStep2Desc_lark: "开放平台 → 你的应用 → 添加应用能力 → 机器人", + prereqStep3Brief_lark: "配置应用权限", + prereqStep3Desc_lark: "为应用开通 wiki:wiki:readonly, drive:drive:readonly, drive:export:readonly, docx:document:readonly 权限", + prereqStep1Brief_feishu_drive: "创建飞书自建应用", + prereqStep1Desc_feishu_drive: "登录飞书开放平台 → 创建企业自建应用", + prereqStep2Brief_feishu_drive: "添加机器人能力", + prereqStep2Desc_feishu_drive: "开放平台 → 你的应用 → 添加应用能力 → 机器人", + prereqStep3Brief_feishu_drive: "配置应用权限", + prereqStep3Desc_feishu_drive: "为应用开通 drive:drive:readonly, drive:export:readonly, docx:document:readonly 权限", + prereqStep1Brief_lark_drive: "创建 Lark 自建应用", + prereqStep1Desc_lark_drive: "登录 Lark 开放平台 → 创建企业自建应用", + prereqStep2Brief_lark_drive: "添加机器人能力", + prereqStep2Desc_lark_drive: "开放平台 → 你的应用 → 添加应用能力 → 机器人", + prereqStep3Brief_lark_drive: "配置应用权限", + prereqStep3Desc_lark_drive: "为应用开通 drive:drive:readonly, drive:export:readonly, docx:document:readonly 权限", prereqBotBrief: '为应用添加「机器人」能力', prereqBotDesc: '开放平台 → 添加应用能力 → 机器人 → 创建版本并发布', prereqPermBrief: '开通 API 权限', @@ -633,7 +657,7 @@ export default { integrationToken: 'Integration Token', apiToken: 'API Token', baseUrl: 'Base URL(可选)', - baseUrlHint: '留空将使用语雀公有云 https://www.yuque.com;如果你使用的是语雀企业版或私有部署,请填写企业域名(例如 https://your-company.yuque.com)', + baseUrlHint: '留空将使用默认公有云地址;如果是私有部署/企业内网部署,或需要通过反向代理访问,请填写自定义地址(例如 https://api-proxy.example.com)', feedUrls: '订阅源地址', feedUrlsHint: '每行一个 RSS / Atom 订阅源地址,支持同时填写多个', authHeaders: '自定义请求头(可选)', @@ -642,6 +666,8 @@ export default { connectorDesc: { feishu: '同步飞书知识库中的文档、表格、文件', lark: '同步 Lark 知识库中的文档、表格、文件(飞书国际版)', + feishu_drive: "同步飞书云盘文件夹中的文档、表格、文件", + lark_drive: "同步 Lark 云盘文件夹中的文档、表格、文件(飞书国际版)", notion: '同步 Notion 中的页面和数据库', yuque: '同步语雀知识库中的文档', rss: '同步 RSS / Atom 订阅源中的文章' @@ -649,6 +675,8 @@ export default { connector: { feishu: '飞书', lark: 'Lark(飞书国际版)', + feishu_drive: "飞书云盘", + lark_drive: "Lark 云盘", notion: 'Notion', yuque: '语雀', rss: 'RSS / Atom 订阅' @@ -696,7 +724,20 @@ export default { syncMode: { incremental: '增量同步', full: '全量同步' - } + }, + drive: { + folderTokenLabel: "云盘文件夹 Token", + folderTokenPlaceholder: "输入 folder_token 或飞书云盘文件夹链接", + folderTokenRequired: "请输入具体文件夹的 folder_token,不支持云空间根目录", + rootNotSupportedHint: "根目录不分页且不返回快捷方式,请选择具体文件夹", + load: "加载", + shareHint: "需先将该云盘文件夹分享给应用所在的群,应用才能访问", + placeholderTitle: "请先加载云盘文件夹", + placeholderDesc: "在上方输入 folder_token(或从飞书云盘文件夹 URL 复制)并点击「加载」", + loadForbiddenHint: "应用无权访问该文件夹。请在飞书云盘中将该文件夹分享给应用所在的群后再试。", + loadAuthHint: "应用凭证无效或缺少云盘权限,请检查 App ID / App Secret 及 drive:drive:readonly 等权限。", + loadNotFoundHint: "folder_token 不存在或已删除,请确认从飞书云盘文件夹 URL 复制的 token 正确。", + }, }, ollama: { unknown: '未知', @@ -5299,6 +5340,8 @@ export default { channelWechat: '微信', channelWecom: '企业微信', channelFeishu: '飞书', + channelFeishuDrive: "飞书云盘", + channelLarkDrive: "Lark 云盘", channelDingtalk: '钉钉', channelSlack: 'Slack', channelIm: 'IM 渠道', diff --git a/frontend/src/utils/markdownDomPurify.ts b/frontend/src/utils/markdownDomPurify.ts index 98354e3cc..7507a1943 100644 --- a/frontend/src/utils/markdownDomPurify.ts +++ b/frontend/src/utils/markdownDomPurify.ts @@ -2,7 +2,7 @@ export const domPurifyForbidTags = ['script', 'style', 'object', 'embed', 'form' export const domPurifyForbidAttr = ['onerror', 'onload', 'onclick', 'onmouseover', 'onfocus', 'onblur'] as const; export const domPurifyAllowedUriRegexp = - /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|blob):|(?:resource|storage|local|minio|cos|tos|s3|oss|ks3|obs):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i; + /^(?:(?:(?:f|ht)tps?|mailto|tel|callto|cid|xmpp|blob):|data:image\/|(?:resource|storage|local|minio|cos|tos|s3|oss|ks3|obs):|[^a-z]|[a-z+.\-]+(?:[^a-z+.\-:]|$))/i; /** Shared DOMPurify security options (FORBID_*, URI scheme, DOM flags). */ export const domPurifySecurityOptions = { diff --git a/frontend/src/views/knowledge/KnowledgeBase.vue b/frontend/src/views/knowledge/KnowledgeBase.vue index 0a61a69b1..1b1963bfc 100644 --- a/frontend/src/views/knowledge/KnowledgeBase.vue +++ b/frontend/src/views/knowledge/KnowledgeBase.vue @@ -594,6 +594,7 @@ const sourceOptions = computed(() => [ { label: t('knowledgeBase.sourceApi'), value: 'api' }, { label: t('knowledgeBase.sourceBrowserExtension'), value: 'browser_extension' }, { label: t('knowledgeBase.channelFeishu'), value: 'feishu' }, + { label: t('knowledgeBase.channelFeishuDrive'), value: 'feishu_drive' }, { label: t('knowledgeBase.channelNotion'), value: 'notion' }, { label: t('knowledgeBase.channelYuque'), value: 'yuque' }, { label: t('knowledgeBase.channelWechat'), value: 'wechat' }, diff --git a/frontend/src/views/knowledge/components/DocumentListView.vue b/frontend/src/views/knowledge/components/DocumentListView.vue index 8e561fe72..dc553565d 100644 --- a/frontend/src/views/knowledge/components/DocumentListView.vue +++ b/frontend/src/views/knowledge/components/DocumentListView.vue @@ -107,6 +107,10 @@ const formatTime = (time?: string) => { const getSourceInfo = (item: KnowledgeItem): { icon: string; label: string } => { const ch = item.channel; if (ch === 'feishu') return { icon: 'cloud-download', label: t('knowledgeBase.channelFeishu') }; + // Drive (云盘) connectors use their own channel so Drive docs show + // "飞书云盘" / "Lark 云盘", distinct from the wiki connector's "飞书". + if (ch === 'feishu_drive') return { icon: 'cloud-download', label: t('knowledgeBase.channelFeishuDrive') }; + if (ch === 'lark_drive') return { icon: 'cloud-download', label: t('knowledgeBase.channelLarkDrive') }; if (ch === 'notion') return { icon: 'cloud-download', label: t('knowledgeBase.channelNotion') }; if (ch === 'yuque') return { icon: 'cloud-download', label: t('knowledgeBase.channelYuque') }; if (ch === 'wechat') return { icon: 'cloud-download', label: t('knowledgeBase.channelWechat') }; diff --git a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue index be0cd7cf3..cd3cf4ccb 100644 --- a/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue +++ b/frontend/src/views/knowledge/settings/DataSourceEditorDialog.vue @@ -196,6 +196,137 @@ const loadingChildrenIds = ref(new Set()) // never needs an extra request. const treeFullyLoaded = ref(false) +// Drive (云盘) root input: the Drive connectors have no "list spaces" API, so +// the user must supply a root folder_token. We collect it here, write it into +// form.config.resource_ids as the single root, then loadResources lists its +// children. See 飞书云盘数据源设计.md §5.2 / ADR-0004. +const driveFolderToken = ref('') +// 必填校验的内联错误文案:非空时输入框显示 error 状态 + 下方 tips, +// 替代全局 MessagePlugin,与表单字段的就地校验风格一致。 +const driveFolderTokenError = ref('') +const driveRootLoaded = ref(false) +const isDriveConnector = (type: string) => type === 'feishu_drive' || type === 'lark_drive' + +// extractDriveFolderToken accepts either a bare folder_token or a Drive folder +// URL (https://xxx.feishu.cn/drive/folder/ or the Lark equivalent +// https://xxx.larksuite.com/drive/folder/) and returns the token. +// Matching is path-based, host-agnostic. Trims surrounding whitespace. +// Returns "" when nothing usable is found. +function extractDriveFolderToken(input: string): string { + const raw = (input || '').trim() + if (!raw) return '' + // Bare token: no scheme, no slash - use as-is. + if (!raw.includes('://') && !raw.includes('/')) return raw + // URL form: extract the segment after /drive/folder/. + const match = raw.match(/\/drive\/folder\/([^/?#]+)/) + if (match && match[1]) return match[1] + // Fallback: last path segment of a URL, or the raw string. + try { + const u = new URL(raw) + const segs = u.pathname.split('/').filter(Boolean) + return segs[segs.length - 1] || raw + } catch { + return raw + } +} + +// loadDriveRoot writes the user-supplied folder_token (or the token extracted +// from a pasted URL) as the root resource_id, then lists the root's children +// so the lazy-load tree can populate. On failure it classifies the error so the +// user gets an actionable hint (e.g. share the folder with the app) instead of +// a raw Feishu error body. +async function loadDriveRoot() { + const token = extractDriveFolderToken(driveFolderToken.value) + if (!token) { + driveFolderTokenError.value = t('datasource.drive.folderTokenRequired') + return + } + driveFolderTokenError.value = '' + // Normalize the input so the user sees the extracted token, not the full URL. + driveFolderToken.value = token + form.value.config.resource_ids = [token] + driveRootLoaded.value = false + loadingResources.value = true + try { + if (!tempDsId.value) { + const res = await createDataSource({ + ...form.value, + knowledge_base_id: props.kbId, + status: 'paused', + } as any) + const created = res?.data || res + tempDsId.value = created.id + } else { + // Edit mode OR a previously-created temp row: persist the new folder_token + // so listResources sees the updated config. Previously this branch skipped + // updates in edit mode, leaving listResources reading the old folder_token. + await updateDataSource(tempDsId.value, { + ...form.value, + knowledge_base_id: props.kbId, + } as any) + } + + const res = await listResources(tempDsId.value) + resources.value = res?.data || res || [] + if (resources.value.length > 0) { + // Mirror loadResources' tree initialization: index parents that already + // arrived with children and auto-expand them. + const parentsWithChildren = new Set() + for (const r of resources.value) { + if (r.parent_id) parentsWithChildren.add(r.parent_id) + } + loadedChildrenIds.value = parentsWithChildren + loadingChildrenIds.value = new Set() + treeFullyLoaded.value = parentsWithChildren.size > 0 + expandedResourceIds.value = new Set( + resources.value + .filter(r => !r.parent_id && r.has_children && parentsWithChildren.has(r.external_id)) + .map(r => r.external_id), + ) + driveRootLoaded.value = true + // In edit mode, reveal pre-existing selections that live below the + // (not-yet-expanded) tree so they are visible and checked - mirrors + // loadResources' behavior for non-Drive connectors. + if (isEdit.value && !treeFullyLoaded.value) { + const loaded = new Set(resources.value.map(r => r.external_id)) + const hidden = selectedResourceIds.value.filter(id => !loaded.has(id)) + if (hidden.length > 0) void revealExistingSelections(hidden) + } + } + } catch (e: any) { + MessagePlugin.error(classifyDriveLoadError(e)) + } + loadingResources.value = false +} + +// classifyDriveLoadError turns a raw Drive list error into an actionable i18n +// message. The Feishu list API returns 403 with code=1061004 when the app has +// not been shared the target folder; without this the user sees "forbidden" +// and has no idea what to do. +function classifyDriveLoadError(e: any): string { + const raw = String(e?.message || e?.error || '') + const lower = raw.toLowerCase() + // 403 / forbidden / 1061004 -> the app lacks access to this specific folder; + // the user must share it with the app's group in Feishu Drive. + if ( + lower.includes('status=403') || + lower.includes('forbidden') || + lower.includes('"code":1061004') || + lower.includes('code=1061004') + ) { + return t('datasource.drive.loadForbiddenHint') + } + // 401 / auth -> app credentials wrong or app lacks the drive scopes. + if (lower.includes('status=401') || lower.includes('auth') || lower.includes('1061005')) { + return t('datasource.drive.loadAuthHint') + } + // Invalid / not-found folder_token. + if (lower.includes('1061003') || lower.includes('not found')) { + return t('datasource.drive.loadNotFoundHint') + } + return raw || t('datasource.resourceLoadFailed') +} + // Shared children/parent indexes — used by tree rendering and selection logic const childrenMap = computed(() => { const map = new Map() @@ -366,6 +497,7 @@ const connectorDefs = computed(() => [ fields: [ { key: 'app_id', labelKey: 'datasource.field.appId', placeholder: 'cli_xxxx' }, { key: 'app_secret', labelKey: 'datasource.field.appSecret', placeholder: '', secret: true }, + { key: 'base_url', labelKey: 'datasource.field.baseUrl', placeholder: 'https://open.feishu.cn', optional: true, hintKey: 'datasource.field.baseUrlHint' }, ], }, { @@ -386,6 +518,45 @@ const connectorDefs = computed(() => [ fields: [ { key: 'app_id', labelKey: 'datasource.field.appId', placeholder: 'cli_xxxx' }, { key: 'app_secret', labelKey: 'datasource.field.appSecret', placeholder: '', secret: true }, + { key: 'base_url', labelKey: 'datasource.field.baseUrl', placeholder: 'https://open.feishu.cn', optional: true, hintKey: 'datasource.field.baseUrlHint' }, + ], + }, + { + // Feishu Drive (云盘) mode: sync documents/files under a user-supplied Drive + // folder_token. Same auth as the wiki connector but no wiki:wiki:readonly + // scope - Drive only needs drive + export + docx. + type: 'feishu_drive', + available: true, + docUrl: 'https://open.feishu.cn/app', + permissionDocUrl: 'https://open.feishu.cn/document/server-docs/docs/drive-v1/file/list', + permissionPageUrl: 'https://open.feishu.cn/app', + requiredPermissions: [ + 'drive:drive:readonly', + 'drive:export:readonly', + 'docx:document:readonly', + ], + fields: [ + { key: 'app_id', labelKey: 'datasource.field.appId', placeholder: 'cli_xxxx' }, + { key: 'app_secret', labelKey: 'datasource.field.appSecret', placeholder: '', secret: true }, + { key: 'base_url', labelKey: 'datasource.field.baseUrl', placeholder: 'https://open.feishu.cn', optional: true, hintKey: 'datasource.field.baseUrlHint' }, + ], + }, + { + // Lark Drive: international counterpart of feishu_drive. + type: 'lark_drive', + available: true, + docUrl: 'https://open.larksuite.com/app', + permissionDocUrl: 'https://open.larksuite.com/document/server-docs/docs/drive-v1/file/list', + permissionPageUrl: 'https://open.larksuite.com/app', + requiredPermissions: [ + 'drive:drive:readonly', + 'drive:export:readonly', + 'docx:document:readonly', + ], + fields: [ + { key: 'app_id', labelKey: 'datasource.field.appId', placeholder: 'cli_xxxx' }, + { key: 'app_secret', labelKey: 'datasource.field.appSecret', placeholder: '', secret: true }, + { key: 'base_url', labelKey: 'datasource.field.baseUrl', placeholder: 'https://open.larksuite.com', optional: true, hintKey: 'datasource.field.baseUrlHint' }, ], }, { @@ -455,6 +626,9 @@ watch(visible, async (v) => { loadedChildrenIds.value = new Set() loadingChildrenIds.value = new Set() treeFullyLoaded.value = false + driveFolderToken.value = '' + driveFolderTokenError.value = '' + driveRootLoaded.value = false rssAuthHeaders.value = [] if (isEdit.value && props.dataSource) { @@ -482,6 +656,18 @@ watch(visible, async (v) => { sync_deletions: props.dataSource.sync_deletions, } selectedResourceIds.value = form.value.config?.resource_ids || [] + // Pre-fill the Drive root folder_token from the saved resource_ids so the + // user sees what they previously entered. driveRootLoaded stays false: the + // tree has not been listed yet, and clicking "load" triggers listResources + // + revealExistingSelections so pre-existing selections are revealed. + if (isDriveConnector(form.value.type)) { + const rids = form.value.config?.resource_ids || [] + if (rids.length > 0) { + // resource_id is "folderToken" or "folderToken:fileToken"; the root is + // the first segment. + driveFolderToken.value = rids[0].split(':')[0] + } + } tempDsId.value = props.dataSource.id } else { replaceCredentialsMode.value = false @@ -760,8 +946,27 @@ async function nextStep() { if ((testResult.value as string) !== 'success') return } } + if (step.value === 2 && isDriveConnector(form.value.type)) { + // folder_token 是 Drive 连接器的必填项:为空就地标错并留在本步, + // 不允许带着空 token 进入同步策略。 + if (!driveFolderToken.value.trim()) { + driveFolderTokenError.value = t('datasource.drive.folderTokenRequired') + return + } + driveFolderTokenError.value = '' + } step.value++ if (step.value === 2) { + // Drive connectors need a user-supplied folder_token before listing. + // In edit mode with a saved folder_token, auto-load so the saved tree + // (and any pre-existing selections) are revealed without an extra click. + // In create mode (no folder_token yet), just show the placeholder. + if (isDriveConnector(form.value.type)) { + if (!driveRootLoaded.value && driveFolderToken.value.trim()) { + void loadDriveRoot() + } + return + } loadResources() } } @@ -963,7 +1168,7 @@ const drawerConfirmText = computed(() => { v-model:visible="visible" :title="drawerTitle" :description="drawerDescription" - :class="form.type ? `datasource-editor-drawer datasource-editor-drawer--${form.type}` : 'datasource-editor-drawer'" + :class="[form.type ? `datasource-editor-drawer datasource-editor-drawer--${form.type}` : 'datasource-editor-drawer', { 'ds-fixed-step': step === 2 }]" :hide-footer="step === 0" :confirm-text="drawerConfirmText" :confirm-loading="submitting || (step === 1 && testing)" @@ -1301,7 +1506,46 @@ const drawerConfirmText = computed(() => {

{{ t('datasource.step.resources') }}

{{ t('datasource.resourceHint') }}

-
+ + +
+ +
+ + + {{ t('datasource.drive.load') }} + +
+
+ + +
+

{{ t('datasource.drive.placeholderTitle') }}

+

{{ t('datasource.drive.placeholderDesc') }}

+
+ +
@@ -1924,6 +2168,80 @@ const drawerConfirmText = computed(() => { color: var(--td-text-color-placeholder); } +/* Drive (云盘) root folder_token input - shown before the lazy-load tree. */ +.drive-folder-input { + display: flex; + flex-direction: column; + gap: 8px; + padding: 12px; + border: 1px solid var(--td-border-level-1-color); + border-radius: 6px; + background: var(--td-bg-color-container); +} + +.drive-folder-input__label { + display: flex; + align-items: center; + gap: 4px; + font-size: 13px; + font-weight: 500; + color: var(--td-text-color-primary); + + /* 与 .form-label.required 一致的红星必填标记 */ + &.required::before { + content: '*'; + color: var(--td-error-color); + font-weight: 500; + line-height: 1; + } +} + +.drive-folder-input__help { + font-size: 15px; + color: var(--td-text-color-placeholder); + cursor: help; + + &:hover { + color: var(--td-text-color-secondary); + } +} + +.drive-folder-input__row { + display: flex; + gap: 8px; + align-items: center; + padding-bottom: 20px +} + +/* Drive tree placeholder: shown before the first successful load. */ +.ds-drive-placeholder { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 6px; + min-height: 120px; + padding: 24px 12px; + border: 1px dashed var(--td-border-level-2-color); + border-radius: 6px; + background: var(--td-bg-color-page); + text-align: center; +} + +.ds-drive-placeholder .ds-empty-title { + margin: 0; + font-size: 13px; + font-weight: 500; + color: var(--td-text-color-primary); +} + +.ds-drive-placeholder .ds-empty-desc { + margin: 0; + font-size: 12px; + line-height: 1.5; + color: var(--td-text-color-placeholder); +} + .resource-picker { display: flex; flex-direction: column; @@ -2289,4 +2607,45 @@ const drawerConfirmText = computed(() => { height: 24px; object-fit: contain; } + +/* Step 2「选择范围」:整步不滚动 —— token 输入区固定,下方资源区域 + (占位 / 加载 / 空态 / 目录树)撑满抽屉剩余高度,树列表内部滚动。 */ +.ds-fixed-step { + .t-drawer__body { + display: flex; + flex-direction: column; + overflow: hidden; + } + + .setting-drawer__body { + flex: 1; + min-height: 0; + } + + .ds-resource-section { + flex: 1; + min-height: 0; + overflow: hidden; + } + + .resource-picker, + .ds-drive-placeholder, + .ds-loading-center, + .ds-resource-empty { + flex: 1; + min-height: 0; + } + + .ds-loading-center { + display: flex; + align-items: center; + justify-content: center; + } + + .resource-picker__list { + flex: 1; + min-height: 0; + max-height: none; + } +} diff --git a/frontend/src/views/knowledge/settings/datasourceIcons.ts b/frontend/src/views/knowledge/settings/datasourceIcons.ts index 78e1c2541..670636f8c 100644 --- a/frontend/src/views/knowledge/settings/datasourceIcons.ts +++ b/frontend/src/views/knowledge/settings/datasourceIcons.ts @@ -7,6 +7,9 @@ import rssIcon from '@/assets/img/datasource-rss.svg' export const datasourceIconMap: Record = { feishu: feishuIcon, lark: larkIcon, + // Drive (云盘) connectors reuse the wiki icons - same product, same brand. + feishu_drive: feishuIcon, + lark_drive: larkIcon, notion: notionIcon, yuque: yuqueIcon, rss: rssIcon, diff --git a/internal/application/service/datasource_service.go b/internal/application/service/datasource_service.go index 931723d03..eba1debea 100644 --- a/internal/application/service/datasource_service.go +++ b/internal/application/service/datasource_service.go @@ -1162,7 +1162,17 @@ func (s *DataSourceService) validateDataSourceConfig(ctx context.Context, ds *ty // // Returns (isUpdate, error) — isUpdate is true when an existing item was replaced. func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource, item *types.FetchedItem, tagIDs []string) (bool, error) { + // Channel decides the knowledge "source" label shown in the UI. Prefer the + // connector-supplied metadata["channel"] (e.g. Feishu Drive sets it to + // "feishu" so Drive docs share the wiki's "飞书" label instead of showing + // "unknown" for the raw ds.Type "feishu_drive"). Fall back to ds.Type so + // connectors that don't set metadata.channel still get a meaningful label. channel := ds.Type // e.g. "feishu", "notion" + if item.Metadata != nil { + if mc, ok := item.Metadata["channel"]; ok && mc != "" { + channel = mc + } + } metadata := map[string]string{ "external_id": item.ExternalID, diff --git a/internal/container/container.go b/internal/container/container.go index b8c2e0873..a5fe47f9f 100644 --- a/internal/container/container.go +++ b/internal/container/container.go @@ -54,7 +54,9 @@ import ( "github.com/Tencent/WeKnora/internal/config" "github.com/Tencent/WeKnora/internal/database" "github.com/Tencent/WeKnora/internal/datasource" - feishuConnector "github.com/Tencent/WeKnora/internal/datasource/connector/feishu" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/drive" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/wiki" notionConnector "github.com/Tencent/WeKnora/internal/datasource/connector/notion" rssConnector "github.com/Tencent/WeKnora/internal/datasource/connector/rss" yuqueConnector "github.com/Tencent/WeKnora/internal/datasource/connector/yuque" @@ -1597,13 +1599,22 @@ func initConnectorRegistry() (*datasource.ConnectorRegistry, error) { registry := datasource.NewConnectorRegistry() var errs error - if err := registry.Register(feishuConnector.NewConnector(feishuConnector.RegionFeishu)); err != nil { + if err := registry.Register(wiki.NewConnector(core.RegionFeishu)); err != nil { errs = errors.Join(errs, fmt.Errorf("register feishu connector: %w", err)) } // Lark is Feishu's international cloud: same connector, different host/tenant. - if err := registry.Register(feishuConnector.NewConnector(feishuConnector.RegionLark)); err != nil { + if err := registry.Register(wiki.NewConnector(core.RegionLark)); err != nil { errs = errors.Join(errs, fmt.Errorf("register lark connector: %w", err)) } + // Feishu/Lark Drive (云盘) mode: different connector type so the registry + // dispatches to the Drive connector. Shares core.Client/Region/export logic + // with the wiki connector. See 飞书云盘数据源设计.md / ADR-0001. + if err := registry.Register(drive.NewDriveConnector(core.RegionFeishuDrive)); err != nil { + errs = errors.Join(errs, fmt.Errorf("register feishu_drive connector: %w", err)) + } + if err := registry.Register(drive.NewDriveConnector(core.RegionLarkDrive)); err != nil { + errs = errors.Join(errs, fmt.Errorf("register lark_drive connector: %w", err)) + } if err := registry.Register(notionConnector.NewConnector()); err != nil { errs = errors.Join(errs, fmt.Errorf("register notion connector: %w", err)) } diff --git a/internal/datasource/connector.go b/internal/datasource/connector.go index 95bc7e929..bce382eb4 100644 --- a/internal/datasource/connector.go +++ b/internal/datasource/connector.go @@ -165,6 +165,22 @@ var ConnectorMetadataRegistry = map[string]ConnectorMetadata{ AuthType: "oauth2", Capabilities: []string{"incremental", "deletion_sync"}, }, + types.ConnectorTypeFeishuDrive: { + Type: types.ConnectorTypeFeishuDrive, + Name: "Feishu Drive (飞书云盘)", + Description: "Sync documents and files from a Feishu Drive folder", + Priority: 0, + AuthType: "oauth2", + Capabilities: []string{"incremental", "deletion_sync"}, + }, + types.ConnectorTypeLarkDrive: { + Type: types.ConnectorTypeLarkDrive, + Name: "Lark Drive", + Description: "Sync documents and files from a Lark Drive folder", + Priority: 0, + AuthType: "oauth2", + Capabilities: []string{"incremental", "deletion_sync"}, + }, types.ConnectorTypeNotion: { Type: types.ConnectorTypeNotion, Name: "Notion", diff --git a/internal/datasource/connector/feishu/connector.go b/internal/datasource/connector/feishu/connector.go deleted file mode 100644 index 04dec5b56..000000000 --- a/internal/datasource/connector/feishu/connector.go +++ /dev/null @@ -1,1152 +0,0 @@ -package feishu - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "maps" - "net/http" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" - "unicode/utf8" - - "github.com/Tencent/WeKnora/internal/datasource" - "github.com/Tencent/WeKnora/internal/logger" - "github.com/Tencent/WeKnora/internal/types" -) - -// Connector implements the datasource.Connector interface for Feishu and, with -// the same code, for Lark: the two clouds expose an identical wiki/docx/drive -// API surface. A Region picks the cloud — see region.go. -type Connector struct { - region Region -} - -// NewConnector creates a connector for the given region (RegionFeishu or RegionLark). -func NewConnector(region Region) *Connector { - return &Connector{region: region} -} - -// Feishu supports resumable streaming sync; the service prefers FetchStream over -// FetchAll/FetchIncremental when a connector implements StreamingConnector. -var _ datasource.StreamingConnector = (*Connector)(nil) - -// Type returns the connector type identifier. -func (c *Connector) Type() string { - return c.region.ConnectorType -} - -const feishuWikiNodeResourceSeparator = ":" - -// feishuStreamCheckpointInterval is how many processed nodes pass between -// cursor checkpoints during a streaming fetch. Small enough that a timed-out -// sync loses little work on resume, large enough that checkpoint persistence -// (a DB write) does not dominate. Overridable in tests. See FetchStream. -var feishuStreamCheckpointInterval = 50 - -// feishuStreamCheckpointMaxInterval bounds checkpointing by wall-clock time as -// well as node count. Without it, a sync of fewer than -// feishuStreamCheckpointInterval very slow (rate-limited) exports could reach -// the 2h task timeout having never checkpointed, and resume from scratch every -// retry — the #2136 "never fully syncs" case. Overridable in tests. -var feishuStreamCheckpointMaxInterval = 30 * time.Second - -// fetchTally accumulates the outcome of fetching a wiki node subtree so the -// connector can emit a single actionable summary. Without it, unsupported nodes -// (mindnote/slides/etc.) vanish with no item, no error and no log, leaving users -// unable to explain why "13 documents synced only 3" (Tencent/WeKnora#2136). -type fetchTally struct { - discovered int - fetched int - failed int - skippedByType map[string]int -} - -func newFetchTally(discovered int) *fetchTally { - return &fetchTally{discovered: discovered, skippedByType: map[string]int{}} -} - -func (t *fetchTally) fetch() { t.fetched++ } -func (t *fetchTally) fail() { t.failed++ } -func (t *fetchTally) skip(objType string) { t.skippedByType[objType]++ } - -func (t *fetchTally) skipped() int { - n := 0 - for _, c := range t.skippedByType { - n += c - } - return n -} - -func (t *fetchTally) summary() string { - return fmt.Sprintf("discovered=%d fetched=%d failed=%d skipped_unsupported=%d by_type=%v", - t.discovered, t.fetched, t.failed, t.skipped(), t.skippedByType) -} - -// Validate verifies that the Feishu configuration is valid by testing connectivity. -func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error { - feishuConfig, err := parseFeishuConfig(config, c.region) - if err != nil { - return err - } - - client := NewClient(feishuConfig) - if err := client.Ping(ctx); err != nil { - return fmt.Errorf("feishu connection failed: %w", err) - } - - return nil -} - -// ListResources lists Feishu Wiki resources for selection, loading the tree -// lazily one level at a time to avoid traversing the entire wiki up front. -// -// - parentID == "" → list all accessible wiki spaces. -// - parentID == spaceID → list the top-level nodes of that space. -// - parentID == "spaceID:nodeToken" → list the direct children of that node. -// -// Eagerly recursing the whole tree here used to time out for large wikis -// (Tencent/WeKnora#1672); the recursive walk now happens only at sync time. -func (c *Connector) ListResources( - ctx context.Context, config *types.DataSourceConfig, parentID string, -) ([]types.Resource, error) { - feishuConfig, err := parseFeishuConfig(config, c.region) - if err != nil { - return nil, err - } - - client := NewClient(feishuConfig) - - if parentID == "" { - spaces, err := client.ListWikiSpaces(ctx) - if err != nil { - return nil, fmt.Errorf("list feishu wiki spaces: %w", err) - } - - resources := make([]types.Resource, 0, len(spaces)) - for _, space := range spaces { - resources = append(resources, types.Resource{ - ExternalID: space.SpaceID, - Name: space.Name, - Type: "wiki_space", - Description: space.Description, - URL: c.region.wikiURL(space.SpaceID), - HasChildren: true, - Metadata: map[string]interface{}{ - "visibility": space.Visibility, - "space_id": space.SpaceID, - }, - }) - } - return resources, nil - } - - // Lazy load: list only the direct children of the given space / node. - spaceID, nodeToken := parseWikiResourceID(parentID) - nodes, err := client.ListWikiNodes(ctx, spaceID, nodeToken) - if err != nil { - return nil, fmt.Errorf("list feishu wiki nodes under %s: %w", parentID, err) - } - - resources := make([]types.Resource, 0, len(nodes)) - for _, node := range nodes { - resources = append(resources, c.wikiNodeToResource(spaceID, node)) - } - return resources, nil -} - -// ResolveResourceAncestors returns the resource IDs of every parent that has to -// be expanded so the lazily-loaded picker can reveal each given selection. For a -// selected node "spaceID:nodeToken" that is its space plus every intermediate -// node up the tree; the walk uses GetWikiNode (parent_node_token) and is O(depth) -// per selection, so it never re-traverses the whole wiki. -func (c *Connector) ResolveResourceAncestors( - ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, -) ([]string, error) { - feishuConfig, err := parseFeishuConfig(config, c.region) - if err != nil { - return nil, err - } - client := NewClient(feishuConfig) - - seen := make(map[string]bool) - ancestors := make([]string, 0) - add := func(id string) { - if id != "" && !seen[id] { - seen[id] = true - ancestors = append(ancestors, id) - } - } - - for _, rid := range resourceIDs { - spaceID, nodeToken := parseWikiResourceID(rid) - if spaceID == "" || nodeToken == "" { - // A space-level selection is already a top-level node in the picker; - // there is nothing above it to reveal. - continue - } - // The space's direct children must be loaded to reveal the top-level node. - add(spaceID) - - // Walk up from the selection to the top, loading each intermediate - // parent so the path down to the selection becomes visible. - current := nodeToken - for current != "" { - node, err := client.GetWikiNode(ctx, spaceID, current) - if err != nil { - // Best-effort: a broken path just stays collapsed, the rest of - // the selections are still revealed. - logger.Warnf(ctx, "[Feishu] resolve ancestors: get node %s:%s: %v", spaceID, current, err) - break - } - if node.ParentNodeID == "" { - break - } - add(makeWikiNodeResourceID(spaceID, node.ParentNodeID)) - current = node.ParentNodeID - } - } - - return ancestors, nil -} - -// FetchAll performs a full sync of all documents from the specified wiki spaces. -func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) { - feishuConfig, err := parseFeishuConfig(config, c.region) - if err != nil { - return nil, err - } - - client := NewClient(feishuConfig) - - var allItems []types.FetchedItem - - for _, resourceID := range resourceIDs { - spaceID, nodeToken := parseWikiResourceID(resourceID) - // List all nodes in this wiki space or selected node subtree recursively - nodes, err := client.ListWikiNodesRecursiveFrom(ctx, spaceID, nodeToken) - if err != nil { - var partialErr *partialWikiNodeListError - if !errors.As(err, &partialErr) { - return nil, fmt.Errorf("list nodes for resource %s: %w", resourceID, err) - } - allItems = appendWikiNodeListFailureItems(allItems, spaceID, resourceID, partialErr.Failures) - } - - // Fetch content for each document node, tallying outcomes so a single - // summary line explains where every discovered node went. - tally := newFetchTally(len(nodes)) - for i, node := range nodes { - items, err := c.fetchNodeContent(ctx, client, node, spaceID, resourceID, config.MultimodalEnabled) - if err != nil { - tally.fail() - // Log error but continue with other nodes - allItems = append(allItems, types.FetchedItem{ - ExternalID: node.NodeToken, - Title: node.Title, - SourceResourceID: resourceID, - Metadata: feishuErrorItemMeta(err, nil), - }) - continue - } - if len(items) > 0 { - tally.fetch() - for _, it := range items { - allItems = append(allItems, *it) - } - } else { - // Unsupported obj_type (mindnote/slides/…): skipped with no item. - tally.skip(node.ObjType) - } - if n := i + 1; n%100 == 0 { - logger.Infof(ctx, "[Feishu] sync progress resource=%s %d/%d (%s)", - resourceID, n, len(nodes), tally.summary()) - } - } - logger.Infof(ctx, "[Feishu] sync summary resource=%s %s", resourceID, tally.summary()) - } - - return allItems, nil -} - -// FetchIncremental performs an incremental sync by comparing node edit times -// against the previously recorded state. -func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) { - feishuConfig, err := parseFeishuConfig(config, c.region) - if err != nil { - return nil, nil, err - } - - client := NewClient(feishuConfig) - - // Parse the previous cursor state - var prevCursor feishuCursor - if cursor != nil && cursor.ConnectorCursor != nil { - cursorBytes, _ := json.Marshal(cursor.ConnectorCursor) - _ = json.Unmarshal(cursorBytes, &prevCursor) - } - - // Build new cursor to track current state - newCursor := feishuCursor{ - LastSyncTime: time.Now(), - SpaceNodeTimes: make(map[string]map[string]string), - } - - var changedItems []types.FetchedItem - - // Get resource IDs from config - resourceIDs := config.ResourceIDs - if len(resourceIDs) == 0 { - return nil, nil, fmt.Errorf("no resource IDs (wiki space IDs or wiki node IDs) configured") - } - - for _, resourceID := range resourceIDs { - spaceID, nodeToken := parseWikiResourceID(resourceID) - // List all nodes in this wiki space or selected node subtree - nodes, err := client.ListWikiNodesRecursiveFrom(ctx, spaceID, nodeToken) - var partialErr *partialWikiNodeListError - if err != nil { - if !errors.As(err, &partialErr) { - return nil, nil, fmt.Errorf("list nodes for resource %s: %w", resourceID, err) - } - changedItems = appendWikiNodeListFailureItems(changedItems, spaceID, resourceID, partialErr.Failures) - } - - newCursor.SpaceNodeTimes[resourceID] = make(map[string]string) - if partialErr != nil && prevCursor.SpaceNodeTimes != nil { - if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok { - for nodeToken, editTime := range prevTimes { - newCursor.SpaceNodeTimes[resourceID][nodeToken] = editTime - } - } - } - - // Build a set of current node tokens for deletion detection - currentNodes := make(map[string]bool) - - for _, node := range nodes { - currentNodes[node.NodeToken] = true - // Use ObjEditTime (document content edit time) for change detection, - // NOT NodeEditTime which only tracks node attribute changes (title, position). - editTimeStr := node.ObjEditTime - if editTimeStr == "" { - editTimeStr = node.NodeEditTime // fallback for nodes that don't have obj_edit_time - } - newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr - - // Check if node has changed since last sync - if prevCursor.SpaceNodeTimes != nil { - if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok { - if prevEditTime, exists := prevTimes[node.NodeToken]; exists { - if prevEditTime == editTimeStr { - // Node unchanged, skip - continue - } - } - } - } - - // Node is new or changed — fetch its content - fetchedItems, err := c.fetchNodeContent(ctx, client, node, spaceID, resourceID, config.MultimodalEnabled) - if err != nil { - // Record failed items - changedItems = append(changedItems, types.FetchedItem{ - ExternalID: node.NodeToken, - Title: node.Title, - SourceResourceID: resourceID, - Metadata: feishuErrorItemMeta(err, nil), - }) - continue - } - for _, it := range fetchedItems { - changedItems = append(changedItems, *it) - } - } - - // Detect deleted nodes - if partialErr == nil && prevCursor.SpaceNodeTimes != nil { - if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok { - for nodeToken := range prevTimes { - if !currentNodes[nodeToken] { - // Node was deleted - changedItems = append(changedItems, types.FetchedItem{ - ExternalID: nodeToken, - IsDeleted: true, - SourceResourceID: resourceID, - }) - } - } - } - } - } - - // Build next sync cursor - nextCursorMap := make(map[string]interface{}) - cursorBytes, _ := json.Marshal(newCursor) - _ = json.Unmarshal(cursorBytes, &nextCursorMap) - - nextSyncCursor := &types.SyncCursor{ - LastSyncTime: time.Now(), - ConnectorCursor: nextCursorMap, - } - - return changedItems, nextSyncCursor, nil -} - -// FetchStream performs a resumable, memory-bounded sync. It unifies the full -// and incremental paths: with cursor == nil it fetches everything, and with a -// cursor it skips nodes whose recorded edit time is unchanged — the same -// mechanism that lets a sync which timed out mid-traversal resume from the last -// checkpoint instead of restarting (Tencent/WeKnora#2136). -// -// Instead of accumulating every item in memory (FetchAll), it Emits each item -// as it is fetched and Checkpoints the cursor every feishuStreamCheckpointInterval -// processed nodes, so progress is durable across the Asynq task's 2h timeout. -func (c *Connector) FetchStream( - ctx context.Context, config *types.DataSourceConfig, - cursor *types.SyncCursor, h datasource.StreamHandler, -) (*types.SyncCursor, error) { - feishuConfig, err := parseFeishuConfig(config, c.region) - if err != nil { - return nil, err - } - client := NewClient(feishuConfig) - - var prevCursor feishuCursor - if cursor != nil && cursor.ConnectorCursor != nil { - cursorBytes, _ := json.Marshal(cursor.ConnectorCursor) - _ = json.Unmarshal(cursorBytes, &prevCursor) - } - - newCursor := feishuCursor{ - LastSyncTime: time.Now(), - SpaceNodeTimes: make(map[string]map[string]string), - } - - resourceIDs := config.ResourceIDs - if len(resourceIDs) == 0 { - return nil, fmt.Errorf("no resource IDs (wiki space IDs or wiki node IDs) configured") - } - - processed := 0 - lastCheckpoint := time.Now() - for _, resourceID := range resourceIDs { - spaceID, nodeToken := parseWikiResourceID(resourceID) - nodes, err := client.ListWikiNodesRecursiveFrom(ctx, spaceID, nodeToken) - var partialErr *partialWikiNodeListError - if err != nil { - if !errors.As(err, &partialErr) { - return nil, fmt.Errorf("list nodes for resource %s: %w", resourceID, err) - } - for _, item := range appendWikiNodeListFailureItems(nil, spaceID, resourceID, partialErr.Failures) { - if eerr := h.Emit(ctx, item); eerr != nil { - return nil, eerr - } - } - } - - newCursor.SpaceNodeTimes[resourceID] = make(map[string]string) - // On a partial listing, carry prior edit times forward so a later full - // listing can still detect changes and deletions. - if partialErr != nil && prevCursor.SpaceNodeTimes != nil { - if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok { - for tok, et := range prevTimes { - newCursor.SpaceNodeTimes[resourceID][tok] = et - } - } - } - - currentNodes := make(map[string]bool) - tally := newFetchTally(len(nodes)) - for i, node := range nodes { - currentNodes[node.NodeToken] = true - editTimeStr := node.ObjEditTime - if editTimeStr == "" { - editTimeStr = node.NodeEditTime - } - - // Prior recorded edit time for this node, if any. - var prevEdit string - var hadPrev bool - if prevCursor.SpaceNodeTimes != nil { - if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok { - prevEdit, hadPrev = prevTimes[node.NodeToken] - } - } - - // Resume/incremental fast-path: a node recorded at its current edit - // time is unchanged (or already synced this run) — keep the record - // and skip re-fetching. - if hadPrev && prevEdit == editTimeStr { - newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr - continue - } - - items, ferr := c.fetchNodeContent(ctx, client, node, spaceID, resourceID, config.MultimodalEnabled) - if ferr != nil { - tally.fail() - // Do NOT advance the cursor: the content was never fetched. - // Retain the prior edit time (if any) so prev != current next - // run and the node is retried, instead of being permanently - // skipped on a transient export failure (Tencent/WeKnora#2136). - if hadPrev { - newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = prevEdit - } - if eerr := h.Emit(ctx, types.FetchedItem{ - ExternalID: node.NodeToken, - Title: node.Title, - SourceResourceID: resourceID, - Metadata: feishuErrorItemMeta(ferr, nil), - }); eerr != nil { - return nil, eerr - } - } else { - // Fetched, or an unsupported obj_type (nothing to fetch): record - // the current edit time so the node is not re-processed next run. - newCursor.SpaceNodeTimes[resourceID][node.NodeToken] = editTimeStr - if len(items) > 0 { - tally.fetch() - for _, it := range items { - if eerr := h.Emit(ctx, *it); eerr != nil { - return nil, eerr - } - } - } else { - // Unsupported obj_type (mindnote/slides/…): no item. - tally.skip(node.ObjType) - } - } - - processed++ - if processed%feishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= feishuStreamCheckpointMaxInterval { - if cerr := h.Checkpoint(ctx, newCursor.toSyncCursor()); cerr != nil { - logger.Warnf(ctx, "[Feishu] stream checkpoint failed: %v", cerr) - } - lastCheckpoint = time.Now() - } - if n := i + 1; n%100 == 0 { - logger.Infof(ctx, "[Feishu] stream progress resource=%s %d/%d (%s)", - resourceID, n, len(nodes), tally.summary()) - } - } - - // Detect deleted nodes (only when the full tree was listed successfully). - if partialErr == nil && prevCursor.SpaceNodeTimes != nil { - if prevTimes, ok := prevCursor.SpaceNodeTimes[resourceID]; ok { - for tok := range prevTimes { - if !currentNodes[tok] { - if eerr := h.Emit(ctx, types.FetchedItem{ - ExternalID: tok, - IsDeleted: true, - SourceResourceID: resourceID, - }); eerr != nil { - return nil, eerr - } - } - } - } - } - logger.Infof(ctx, "[Feishu] stream summary resource=%s %s", resourceID, tally.summary()) - } - - return newCursor.toSyncCursor(), nil -} - -// toSyncCursor converts the connector-specific feishuCursor into the generic -// SyncCursor persisted by the service. It marshals through JSON so the returned -// value is a snapshot, decoupled from later mutation of the connector's maps. -func (fc feishuCursor) toSyncCursor() *types.SyncCursor { - m := make(map[string]interface{}) - cursorBytes, _ := json.Marshal(fc) - _ = json.Unmarshal(cursorBytes, &m) - return &types.SyncCursor{ - LastSyncTime: fc.LastSyncTime, - ConnectorCursor: m, - } -} - -var reFeishuErrorCode = regexp.MustCompile(`code["\s]*[:=]\s*(\d+)`) - -// feishuErrorCode extracts the numeric Feishu error code from a raw error string -// (e.g. `body={"code":1663,...}` or `code=1663`), best-effort. -func feishuErrorCode(raw string) string { - if m := reFeishuErrorCode.FindStringSubmatch(raw); len(m) == 2 { - return m[1] - } - return "" -} - -// feishuFailure classifies a raw connector/API error into a stable i18n code -// (mapped to a localized string on the frontend), an optional numeric Feishu -// error code for interpolation, and an English fallback message for clients -// without the i18n key. The raw status/JSON body/log_id is never returned here — -// it stays in the server logs. Dumping it in the UI is the anti-pattern -// Airbyte/Fivetran/Onyx warn against. Transient errors are retried next sync -// (the cursor is retained); auth/permission errors point at the fix instead. -func feishuFailure(err error) (code, codeValue, fallback string) { - if err == nil { - return "sync_failed", "", "Sync failed; will retry on the next sync" - } - s := strings.ToLower(err.Error()) - - switch { - case strings.Contains(s, "auth error"), - strings.Contains(s, "invalid access token"), - strings.Contains(s, "permission"), - strings.Contains(s, "forbidden"), - strings.Contains(s, "status=403"): - return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes" - case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"): - return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync" - case strings.Contains(s, "timed out"), - strings.Contains(s, "timeout"), - strings.Contains(s, "deadline exceeded"): - return "feishu_timeout", "", "Export or request timed out; will retry on the next sync" - case strings.Contains(s, "server error"): - return "feishu_server_unavailable", "", "Feishu service temporarily unavailable; will retry on the next sync" - case strings.Contains(s, "api error"), - strings.Contains(s, "export task failed"), - strings.Contains(s, "download failed"): - if v := feishuErrorCode(err.Error()); v != "" { - return "feishu_api_error", v, fmt.Sprintf("Feishu API error (code=%s); will retry on the next sync", v) - } - return "feishu_api_error_generic", "", "Feishu API error; will retry on the next sync" - default: - return "sync_failed", "", "Sync failed; will retry on the next sync" - } -} - -// feishuErrorItemMeta builds the metadata for a failed item: the raw error (for -// server logs) plus the classified i18n code / param / fallback (for a -// localisable SyncItemError in the UI), merged with any caller-supplied extras. -func feishuErrorItemMeta(err error, extra map[string]string) map[string]string { - code, codeValue, fallback := feishuFailure(err) - m := map[string]string{ - "error": err.Error(), - "error_reason_code": code, - "error_reason": fallback, - } - if codeValue != "" { - m["error_reason_code_value"] = codeValue - } - for k, v := range extra { - m[k] = v - } - return m -} - -func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, resourceID string, failures []wikiNodeListFailure) []types.FetchedItem { - for _, failure := range failures { - node := failure.Node - title := node.Title - if title == "" { - title = node.NodeToken - } - items = append(items, types.FetchedItem{ - ExternalID: node.NodeToken, - Title: title, - SourceResourceID: resourceID, - Metadata: feishuErrorItemMeta(failure.Err, map[string]string{ - "channel": types.ChannelFeishu, - "node_token": node.NodeToken, - "space_id": spaceID, - "failure_stage": "list_children", - }), - }) - } - return items -} - -// parseableAttachmentExts are attachment extensions worth ingesting as their -// own knowledge entries; other files (icons, tiny decor) are skipped. -var parseableAttachmentExts = map[string]bool{ - ".pdf": true, ".doc": true, ".docx": true, ".xls": true, ".xlsx": true, - ".ppt": true, ".pptx": true, ".txt": true, ".md": true, ".csv": true, -} - -// minAttachmentBytes filters out decorative micro-files. -const minAttachmentBytes = 2 * 1024 - -// fetchNodeContent fetches the content of a single wiki node and converts it to a -// slice of FetchedItems. For docx nodes it fans out into a main Markdown document -// plus optional attachment sub-items. Dispatches to different retrieval strategies -// based on obj_type: -// - docx → blocks API (Markdown) with export fallback; may return attachments -// - doc/sheet/bitable → export API → binary file -// - file → drive download → original file (PDF/Word/image/etc.) -// - mindnote → skip (no API) -// - slides → skip (no API) -func (c *Connector) fetchNodeContent(ctx context.Context, client *Client, node wikiNode, spaceID string, resourceID string, multimodalEnabled bool) ([]*types.FetchedItem, error) { - if !isSupportedDocType(node.ObjType) { - return nil, nil - } - - editTime := parseFeishuTimestamp(node.NodeEditTime) - baseMeta := map[string]string{ - "obj_token": node.ObjToken, - "obj_type": node.ObjType, - "node_token": node.NodeToken, - "space_id": spaceID, - "creator": node.Creator, - "owner": node.Owner, - "channel": types.ChannelFeishu, - } - - switch node.ObjType { - case "docx": - return c.fetchDocxWithBlocks(ctx, client, node, resourceID, editTime, baseMeta, multimodalEnabled) - case "doc", "sheet", "bitable": - item, err := c.fetchViaExport(ctx, client, node, resourceID, editTime, baseMeta) - if err != nil { - return nil, err - } - return []*types.FetchedItem{item}, nil - case "file": - item, err := c.fetchDriveFile(ctx, client, node, resourceID, editTime, baseMeta) - if err != nil { - return nil, err - } - return []*types.FetchedItem{item}, nil - default: - return nil, nil - } -} - -// fetchViaExport exports a doc/sheet/bitable node via the async export API and -// returns a single FetchedItem containing the exported binary. -func (c *Connector) fetchViaExport(ctx context.Context, client *Client, node wikiNode, resourceID string, editTime time.Time, baseMeta map[string]string) (*types.FetchedItem, error) { - // Export as a file via the async export API - data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType) - if err != nil { - return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err) - } - - // Ensure a reasonable file name with correct extension - ext := exportFileExtToSuffix[objTypeToExportFileExtension[node.ObjType]] - if fileName == "" { - fileName = sanitizeFileName(node.Title) + ext - } else if !strings.HasSuffix(strings.ToLower(fileName), ext) { - // Feishu often returns the doc title without extension — append it - fileName = sanitizeFileName(fileName) + ext - } - - return &types.FetchedItem{ - ExternalID: node.NodeToken, - Title: node.Title, - Content: data, - ContentType: "application/octet-stream", - FileName: fileName, - URL: c.region.wikiURL(node.NodeToken), - UpdatedAt: editTime, - SourceResourceID: resourceID, - Metadata: baseMeta, - }, nil -} - -// fetchDriveFile downloads an original uploaded file from Drive and returns a -// single FetchedItem containing the raw bytes. -func (c *Connector) fetchDriveFile(ctx context.Context, client *Client, node wikiNode, resourceID string, editTime time.Time, baseMeta map[string]string) (*types.FetchedItem, error) { - // Download the original uploaded file from Drive - data, err := client.DownloadDriveFile(ctx, node.ObjToken) - if err != nil { - return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err) - } - - // Use the node title as file name; it usually preserves the original extension - fileName := node.Title - if fileName == "" { - fileName = node.ObjToken - } - - return &types.FetchedItem{ - ExternalID: node.NodeToken, - Title: node.Title, - Content: data, - ContentType: "application/octet-stream", - FileName: fileName, - URL: c.region.wikiURL(node.NodeToken), - UpdatedAt: editTime, - SourceResourceID: resourceID, - Metadata: baseMeta, - }, nil -} - -// fetchDocxWithBlocks retrieves a docx node via the blocks API, converts it to -// Markdown, and returns a main item plus any parseable attachment sub-items. -// Falls back to the export API if the blocks API returns an error. -func (c *Connector) fetchDocxWithBlocks(ctx context.Context, client *Client, node wikiNode, - resourceID string, editTime time.Time, baseMeta map[string]string, multimodalEnabled bool) ([]*types.FetchedItem, error) { - - blocks, err := client.ListDocumentBlocks(ctx, node.ObjToken) - if err != nil { - logger.Warnf(ctx, "[Feishu] blocks API failed for %s (%s), falling back to export: %v", - node.Title, node.ObjToken, err) - item, ferr := c.fetchViaExport(ctx, client, node, resourceID, editTime, baseMeta) - if ferr != nil { - return nil, ferr - } - // Do NOT set ReplacesSubtree here. The export path cannot re-enumerate the - // doc's attachments, so it emits no sub-items. If the blocks API failed - // transiently (rate limit / 5xx), sweeping would delete the good attachment - // children from the prior blocks-path sync with nothing to replace them — - // silent data loss. Leaving stale children is the safe choice; they are - // reconciled on the next successful blocks-path sync. - return []*types.FetchedItem{item}, nil - } - - md, atts, err := blocksToMarkdown(ctx, client, blocks) - if err != nil { - return nil, fmt.Errorf("convert blocks %s: %w", node.Title, err) - } - - // A docx that renders to empty Markdown (a blank page, or only block types that - // produce no text) must NOT be emitted as a main item with empty Content plus a - // wiki URL: ingestItem would then take its URL branch and CreateKnowledgeFromURL - // against the login-gated Feishu page — a guaranteed failure that reports the - // node as Failed. Fall back to the export path, which always yields a valid (if - // minimal) .docx binary, preserving the invariant that a supported docx node - // ingests as content bytes. An empty render implies no File/Image blocks either - // (both write a placeholder into the Markdown), so no attachment/image sub-items - // are lost by skipping the downdrill here. - if len(strings.TrimSpace(string(md))) == 0 { - logger.Infof(ctx, "[Feishu] doc %s (%s): blocks rendered empty Markdown, falling back to export", - node.Title, node.ObjToken) - item, ferr := c.fetchViaExport(ctx, client, node, resourceID, editTime, baseMeta) - if ferr != nil { - return nil, ferr - } - return []*types.FetchedItem{item}, nil - } - - main := &types.FetchedItem{ - ExternalID: node.NodeToken, - Title: node.Title, - Content: md, - ContentType: "text/markdown", - FileName: sanitizeFileName(node.Title) + ".md", - URL: c.region.wikiURL(node.NodeToken), - UpdatedAt: editTime, - SourceResourceID: resourceID, - Metadata: baseMeta, - ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync - } - items := []*types.FetchedItem{main} - - // keep collects the external_id of every attachment still present in the doc - // this sync — parseable or not, downloaded or not. The subtree sweep deletes - // only children NOT in this set, so an attachment that is still present but - // could not be re-ingested this cycle (unclassifiable filename, transient - // download failure) keeps its previously-synced good copy instead of being - // deleted with nothing to replace it. Only attachments genuinely removed from - // the doc fall out of keep and get swept — one bad attachment no longer - // freezes reconciliation of its siblings. - keep := make([]string, 0, len(atts)) - childMeta := func() map[string]string { - m := maps.Clone(baseMeta) - m["parent_node_token"] = node.NodeToken - m["attachment"] = "true" - return m - } - for _, a := range atts { - childID := types.SubtreeChildID(node.NodeToken, "file", a.FileToken) - keep = append(keep, childID) // present in the doc → never sweep as stale - ext := strings.ToLower(filepath.Ext(a.Name)) - if ext == "" { - // No filename extension to classify by: we can neither decide whether - // the file is parseable nor build a valid typed filename downstream, so - // it cannot be ingested as a standalone item this cycle. Log instead of - // dropping silently; its prior copy (if any) is preserved via keep. - logger.Warnf(ctx, "[Feishu] doc %s: skipping attachment with no usable filename (token=%s name=%q)", - node.ObjToken, a.FileToken, a.Name) - continue - } - if !parseableAttachmentExts[ext] { - continue // decorative/non-parseable → not a standalone knowledge item - } - data, derr := client.DownloadMediaFile(ctx, a.FileToken) - if derr != nil { - // Degrade gracefully: a single failed attachment (revoked token, - // permission gap, transient error) must NOT discard the already-built - // document body and its other attachments. Surface it as a per-item - // sync error (visible in the UI and counted, not just a server log); - // keep already preserves its prior copy so the sweep won't delete it. - logger.Warnf(ctx, "[Feishu] doc %s: attachment %q (token=%s) download failed: %v", - node.ObjToken, a.Name, a.FileToken, derr) - items = append(items, &types.FetchedItem{ - ExternalID: childID, - Title: a.Name, - SourceResourceID: resourceID, - Metadata: feishuErrorItemMeta(derr, childMeta()), - }) - continue - } - if len(data) < minAttachmentBytes { - // Below the decorative-micro-file floor. Log rather than drop silently - // (the sibling skip paths above also surface a note); its prior copy, if - // any, is preserved via keep. - logger.Infof(ctx, "[Feishu] doc %s: skipping tiny attachment %q (token=%s, %d bytes < %d)", - node.ObjToken, a.Name, a.FileToken, len(data), minAttachmentBytes) - continue - } - items = append(items, &types.FetchedItem{ - ExternalID: childID, - Title: a.Name, - Content: data, - ContentType: "application/octet-stream", - FileName: sanitizeFileName(a.Name), - URL: c.region.wikiURL(node.NodeToken), - UpdatedAt: editTime, - SourceResourceID: resourceID, - Metadata: childMeta(), - }) - } - - // Embedded images: the Markdown body only carries a token-free placeholder for - // each image, so image-borne text (screenshots/diagrams) would be unsearchable. - // Emit each image as a standalone sub-item whose bytes flow through WeKnora's - // VLM OCR+caption pipeline, recovering that text. The image's external_id is - // ALWAYS added to keep (so toggling VLM off later does not sweep previously - // OCR'd images), but the bytes are only downloaded and ingested when the KB has - // multimodal enabled — ingesting an image into a non-VLM KB is rejected, so - // doing so would turn every image into a failed sync item. - imgMeta := func() map[string]string { - m := maps.Clone(baseMeta) - m["parent_node_token"] = node.NodeToken - m["embedded_image"] = "true" - return m - } - for _, b := range blocks { - if b.BlockType != blockTypeImage || b.Image == nil || b.Image.Token == "" { - continue - } - childID := types.SubtreeChildID(node.NodeToken, "image", b.Image.Token) - keep = append(keep, childID) // present in the doc → never sweep as stale - if !multimodalEnabled { - continue // KB can't OCR images; the inline placeholder is all we keep - } - data, derr := client.DownloadMediaFile(ctx, b.Image.Token) - if derr != nil { - // A failed image download (revoked token, permission gap, transient - // error) is a genuine fetch failure, not the best-effort ingest - // rejection a non-VLM KB produces — surface it as a visible per-item - // sync error exactly like a failed attachment download, instead of - // dropping it to a server log only. keep already preserves any prior - // OCR'd copy so the sweep won't delete it. - logger.Warnf(ctx, "[Feishu] doc %s: image (token=%s) download failed: %v", - node.ObjToken, b.Image.Token, derr) - items = append(items, &types.FetchedItem{ - ExternalID: childID, - Title: fmt.Sprintf("%s(内嵌图片)", node.Title), - SourceResourceID: resourceID, - Metadata: feishuErrorItemMeta(derr, imgMeta()), - }) - continue - } - if len(data) < minAttachmentBytes { - continue // decorative micro-image (icon/spacer) - } - ext, contentType, ok := supportedImageExt(data) - if !ok { - logger.Warnf(ctx, "[Feishu] doc %s: skipping image (token=%s) of unsupported type %q", - node.ObjToken, b.Image.Token, contentType) - continue - } - items = append(items, &types.FetchedItem{ - ExternalID: childID, - Title: fmt.Sprintf("%s(内嵌图片)", node.Title), - Content: data, - ContentType: contentType, - FileName: "image-" + b.Image.Token + ext, - URL: c.region.wikiURL(node.NodeToken), - UpdatedAt: editTime, - SourceResourceID: resourceID, - Metadata: imgMeta(), - }) - } - - // Reconcile the subtree against the attachments still present in the doc: the - // sweep (in datasource_service) deletes only prior children absent from keep. - main.SubtreeKeep = keep - return items, nil -} - -// supportedImageExt sniffs image bytes and returns the filename extension and -// content type WeKnora accepts for a standalone image knowledge item (png/jpg/ -// gif — the image set isValidFileType admits). ok is false for non-image or -// unsupported formats (e.g. webp/bmp), which the caller skips rather than -// mislabel — a wrong extension would fail parsing. The detected content type is -// returned even when ok is false so the caller can log it without re-sniffing. -func supportedImageExt(data []byte) (ext, contentType string, ok bool) { - switch ct := http.DetectContentType(data); ct { - case "image/png": - return ".png", ct, true - case "image/jpeg": - return ".jpg", ct, true - case "image/gif": - return ".gif", ct, true - default: - return "", ct, false - } -} - -// --- Helper functions --- - -func makeWikiNodeResourceID(spaceID, nodeToken string) string { - return spaceID + feishuWikiNodeResourceSeparator + nodeToken -} - -func parseWikiResourceID(resourceID string) (spaceID string, nodeToken string) { - spaceID, nodeToken, _ = strings.Cut(resourceID, feishuWikiNodeResourceSeparator) - return spaceID, nodeToken -} - -func (c *Connector) wikiNodeToResource(spaceID string, node wikiNode) types.Resource { - parentID := spaceID - if node.ParentNodeID != "" { - parentID = makeWikiNodeResourceID(spaceID, node.ParentNodeID) - } - - name := node.Title - if name == "" { - name = node.NodeToken - } - - modifiedAt := parseFeishuTimestamp(node.ObjEditTime) - if modifiedAt.IsZero() { - modifiedAt = parseFeishuTimestamp(node.NodeEditTime) - } - - return types.Resource{ - ExternalID: makeWikiNodeResourceID(spaceID, node.NodeToken), - Name: name, - Type: "wiki_node", - URL: c.region.wikiURL(node.NodeToken), - ParentID: parentID, - HasChildren: node.HasChild, - ModifiedAt: modifiedAt, - Metadata: map[string]interface{}{ - "space_id": spaceID, - "node_token": node.NodeToken, - "obj_token": node.ObjToken, - "obj_type": node.ObjType, - }, - } -} - -// parseFeishuConfig extracts and validates Feishu/Lark-specific configuration. -// -// base_url stays an explicit override so existing data sources that pointed a -// "feishu" connector at open.larksuite.com keep working; when it is unset the -// region's own host is filled in, making the resolved Config.BaseURL concrete -// for everything downstream. -func parseFeishuConfig(config *types.DataSourceConfig, region Region) (*Config, error) { - if config == nil { - return nil, fmt.Errorf("config is nil") - } - - credBytes, err := json.Marshal(config.Credentials) - if err != nil { - return nil, fmt.Errorf("marshal credentials: %w", err) - } - - var feishuConfig Config - if err := json.Unmarshal(credBytes, &feishuConfig); err != nil { - return nil, fmt.Errorf("parse %s credentials: %w", region.ConnectorType, err) - } - - if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" { - return nil, fmt.Errorf("%s app_id and app_secret are required", region.ConnectorType) - } - - if feishuConfig.BaseURL == "" { - feishuConfig.BaseURL = region.OpenBaseURL - } - - // Timezone is a display setting (bitable date rendering), not a credential, so - // it lives in Settings. Empty falls back to GMT+8 in resolveLocation. - if feishuConfig.Timezone == "" && config.Settings != nil { - if tz, ok := config.Settings["timezone"].(string); ok { - feishuConfig.Timezone = strings.TrimSpace(tz) - } - } - - if err := datasource.ValidateConnectorBaseURL(feishuConfig.GetBaseURL()); err != nil { - return nil, err - } - - return &feishuConfig, nil -} - -// isSupportedDocType checks if a Feishu document type can be synced. -// mindnote and slides have no content read API and are skipped. -func isSupportedDocType(objType string) bool { - switch objType { - case "docx", "doc", "sheet", "bitable", "file": - return true - default: - // mindnote, slides — no content retrieval API available - return false - } -} - -// parseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time. -func parseFeishuTimestamp(ts string) time.Time { - if ts == "" { - return time.Time{} - } - sec, err := strconv.ParseInt(ts, 10, 64) - if err != nil { - return time.Time{} - } - return time.Unix(sec, 0) -} - -// sanitizeFileName removes characters that are invalid in filenames and -// truncates at a UTF-8 rune boundary. Raw byte truncation would split a -// multi-byte codepoint (Chinese chars are 3 bytes) and produce invalid UTF-8 -// that downstream validation (utf8.ValidString) rejects. -// -// The extension is preserved across truncation: only the base name is trimmed, -// so a long attachment name like "很长的名字….pdf" keeps its ".pdf" suffix that -// downstream file-type classification depends on. -func sanitizeFileName(name string) string { - if name == "" { - return "untitled" - } - replacer := strings.NewReplacer( - "/", "_", "\\", "_", ":", "_", "*", "_", - "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", - ) - result := replacer.Replace(name) - const maxBytes = 200 - if len(result) <= maxBytes { - return result - } - ext := filepath.Ext(result) - if len(ext) >= maxBytes { - ext = "" // pathological: extension alone overflows the budget → drop it - } - base := truncateUTF8(result[:len(result)-len(ext)], maxBytes-len(ext)) - return base + ext -} - -// truncateUTF8 shortens s to at most maxBytes bytes without splitting a -// multi-byte rune: after a hard byte cut it trims any trailing partial codepoint. -func truncateUTF8(s string, maxBytes int) string { - if len(s) <= maxBytes { - return s - } - s = s[:maxBytes] - for len(s) > 0 { - r, size := utf8.DecodeLastRuneInString(s) - if r != utf8.RuneError || size != 1 { - break - } - s = s[:len(s)-1] - } - return s -} diff --git a/internal/datasource/connector/feishu/blocks.go b/internal/datasource/connector/feishu/core/blocks.go similarity index 69% rename from internal/datasource/connector/feishu/blocks.go rename to internal/datasource/connector/feishu/core/blocks.go index 48cee78b8..8449b51cf 100644 --- a/internal/datasource/connector/feishu/blocks.go +++ b/internal/datasource/connector/feishu/core/blocks.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "context" @@ -15,101 +15,116 @@ import ( // Feishu docx block_type integer enum (subset this connector handles). // https://open.feishu.cn/document/server-docs/docs/docs/docx-v1/docx-structure const ( - blockTypePage = 1 - blockTypeText = 2 - blockTypeHeading1 = 3 + BlockTypePage = 1 + BlockTypeText = 2 + BlockTypeHeading1 = 3 blockTypeHeading9 = 11 - blockTypeBullet = 12 - blockTypeOrdered = 13 - blockTypeCode = 14 - blockTypeQuote = 15 - blockTypeTodo = 17 - blockTypeBitable = 18 - blockTypeCallout = 19 - blockTypeDivider = 22 - blockTypeFile = 23 - blockTypeImage = 27 - blockTypeSheet = 30 - blockTypeTable = 31 - blockTypeTableCell = 32 + BlockTypeBullet = 12 + BlockTypeOrdered = 13 + BlockTypeCode = 14 + BlockTypeQuote = 15 + BlockTypeTodo = 17 + BlockTypeBitable = 18 + BlockTypeCallout = 19 + BlockTypeDivider = 22 + BlockTypeFile = 23 + BlockTypeImage = 27 + BlockTypeSheet = 30 + BlockTypeTable = 31 + BlockTypeTableCell = 32 ) // maxDocumentBlocks caps how many blocks a single document contributes, guarding // against pathological/adversarial documents. Far above any real Feishu doc. const maxDocumentBlocks = 50000 -// textElement is one inline run inside a text-bearing block. -type textElement struct { - TextRun *struct { - Content string `json:"content"` - } `json:"text_run"` +// TextElement is one inline run inside a text-bearing block. +// TextRun is the text_run payload of a TextElement. +type TextRun struct { + Content string `json:"content"` } -// blockText is the shared shape of text-bearing blocks (text, headingN, bullet…). -type blockText struct { - Elements []textElement `json:"elements"` +type TextElement struct { + TextRun *TextRun `json:"text_run"` } -// docxBlock is one node in the flat block array returned by the blocks API. -type docxBlock struct { +// BlockText is the shared shape of text-bearing blocks (text, headingN, bullet…). +type BlockText struct { + Elements []TextElement `json:"elements"` +} + +// BlockTokenRef is the shared shape of sheet/bitable/image block payloads +// (the JSON tag differs per field; only the token matters). +type BlockTokenRef struct { + Token string `json:"token"` +} + +// BlockFileRef is the file block payload: the attachment token plus its name. +type BlockFileRef struct { + Token string `json:"token"` + Name string `json:"name"` +} + +// BlockTableProperty carries the table grid shape. +type BlockTableProperty struct { + ColumnSize int `json:"column_size"` +} + +// BlockTable is the table block payload: cell block IDs plus grid property. +type BlockTable struct { + Cells []string `json:"cells"` + Property *BlockTableProperty `json:"property"` +} + +// DocxBlock is one node in the flat block array returned by the blocks API. +type DocxBlock struct { BlockID string `json:"block_id"` ParentID string `json:"parent_id"` BlockType int `json:"block_type"` Children []string `json:"children"` - Text *blockText `json:"text"` - Heading1 *blockText `json:"heading1"` - Heading2 *blockText `json:"heading2"` - Heading3 *blockText `json:"heading3"` - Heading4 *blockText `json:"heading4"` - Heading5 *blockText `json:"heading5"` - Heading6 *blockText `json:"heading6"` - Heading7 *blockText `json:"heading7"` - Heading8 *blockText `json:"heading8"` - Heading9 *blockText `json:"heading9"` - Bullet *blockText `json:"bullet"` - Ordered *blockText `json:"ordered"` - Code *blockText `json:"code"` - Quote *blockText `json:"quote"` - Todo *blockText `json:"todo"` - Callout *blockText `json:"callout"` + Text *BlockText `json:"text"` + Heading1 *BlockText `json:"heading1"` + Heading2 *BlockText `json:"heading2"` + Heading3 *BlockText `json:"heading3"` + Heading4 *BlockText `json:"heading4"` + Heading5 *BlockText `json:"heading5"` + Heading6 *BlockText `json:"heading6"` + Heading7 *BlockText `json:"heading7"` + Heading8 *BlockText `json:"heading8"` + Heading9 *BlockText `json:"heading9"` + Bullet *BlockText `json:"bullet"` + Ordered *BlockText `json:"ordered"` + Code *BlockText `json:"code"` + Quote *BlockText `json:"quote"` + Todo *BlockText `json:"todo"` + Callout *BlockText `json:"callout"` - Sheet *struct { - Token string `json:"token"` - } `json:"sheet"` - Bitable *struct { - Token string `json:"token"` - } `json:"bitable"` - File *struct { - Token string `json:"token"` - Name string `json:"name"` - } `json:"file"` - Image *struct { - Token string `json:"token"` - } `json:"image"` + Sheet *BlockTokenRef `json:"sheet"` + Bitable *BlockTokenRef `json:"bitable"` + File *BlockFileRef `json:"file"` + Image *BlockTokenRef `json:"image"` - Table *struct { - Cells []string `json:"cells"` - Property *struct { - ColumnSize int `json:"column_size"` - } `json:"property"` - } `json:"table"` + Table *BlockTable `json:"table"` } -// docxBlocksResponse is the response for GET .../documents/:id/blocks. -type docxBlocksResponse struct { - apiResponse - Data struct { - Items []docxBlock `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - } `json:"data"` +// DocxBlocksData is the data payload of DocxBlocksResponse. +type DocxBlocksData struct { + Items []DocxBlock `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` } -// ListDocumentBlocks returns every block of a docx document as a flat, +// DocxBlocksResponse is the response for GET .../documents/:id/blocks. +type DocxBlocksResponse struct { + ApiResponse + Data DocxBlocksData `json:"data"` +} + +// listDocumentBlocks returns every block of a docx document as a flat, // pre-order array. Paginates at 500 blocks/page. documentID is the obj_token. -func (c *Client) ListDocumentBlocks(ctx context.Context, documentID string) ([]docxBlock, error) { - var all []docxBlock +func (c *Client) listDocumentBlocks(ctx context.Context, documentID string) ([]DocxBlock, error) { + var all []DocxBlock pageToken := "" for { path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks?page_size=500&document_revision_id=-1", @@ -117,8 +132,8 @@ func (c *Client) ListDocumentBlocks(ctx context.Context, documentID string) ([]d if pageToken != "" { path += "&page_token=" + url.QueryEscape(pageToken) } - var resp docxBlocksResponse - if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + var resp DocxBlocksResponse + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { return nil, fmt.Errorf("list document blocks: %w", err) } if resp.Code != 0 { @@ -148,21 +163,27 @@ func (c *Client) ListDocumentBlocks(ctx context.Context, documentID string) ([]d // is truncated and the caller annotates the omission. const maxTableRows = 500 -// sheetValuesResponse is the response for sheets-v2 values read. -type sheetValuesResponse struct { - apiResponse - Data struct { - ValueRange struct { - Values [][]any `json:"values"` - } `json:"valueRange"` - } `json:"data"` +// sheetValueRange is the valueRange payload of sheetValuesData. +type sheetValueRange struct { + Values [][]any `json:"values"` } -// ReadSheetRange reads the cell values of an embedded spreadsheet block. +// sheetValuesData is the data payload of sheetValuesResponse. +type sheetValuesData struct { + ValueRange sheetValueRange `json:"valueRange"` +} + +// sheetValuesResponse is the response for sheets-v2 values read. +type sheetValuesResponse struct { + ApiResponse + Data sheetValuesData `json:"data"` +} + +// readSheetRange reads the cell values of an embedded spreadsheet block. // embedToken is the block's sheet.token, formatted "spreadsheetToken_sheetId". // Cells are stringified (display value) for RAG text retrieval. Rows are capped // at maxTableRows; truncated is true when the source had more rows than that. -func (c *Client) ReadSheetRange(ctx context.Context, embedToken string) ([][]string, bool, error) { +func (c *Client) readSheetRange(ctx context.Context, embedToken string) ([][]string, bool, error) { idx := strings.LastIndex(embedToken, "_") if idx < 0 { return nil, false, fmt.Errorf("invalid sheet embed token: %q", embedToken) @@ -171,7 +192,7 @@ func (c *Client) ReadSheetRange(ctx context.Context, embedToken string) ([][]str path := fmt.Sprintf("/open-apis/sheets/v2/spreadsheets/%s/values/%s?valueRenderOption=ToString", url.PathEscape(spreadsheetToken), url.PathEscape(sheetID)) var resp sheetValuesResponse - if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { return nil, false, fmt.Errorf("read sheet range: %w", err) } if resp.Code != 0 { @@ -293,20 +314,29 @@ func bitableCellToString(v any) string { } } +// bitableFieldProperty carries a field's type-specific settings. +type bitableFieldProperty struct { + // DateFormatter distinguishes a date-only column from a datetime one. + DateFormatter string `json:"date_formatter"` +} + +// bitableField is one entry of bitableFieldsData.Items. +type bitableField struct { + FieldName string `json:"field_name"` + Type int `json:"type"` + Property *bitableFieldProperty `json:"property"` +} + +// bitableFieldsData is the data payload of bitableFieldsResponse. +type bitableFieldsData struct { + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` + Items []bitableField `json:"items"` +} + type bitableFieldsResponse struct { - apiResponse - Data struct { - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - Items []struct { - FieldName string `json:"field_name"` - Type int `json:"type"` - Property *struct { - // DateFormatter distinguishes a date-only column from a datetime one. - DateFormatter string `json:"date_formatter"` - } `json:"property"` - } `json:"items"` - } `json:"data"` + ApiResponse + Data bitableFieldsData `json:"data"` } // maxBitableFieldPageSize is the documented per-page cap for the bitable @@ -314,22 +344,28 @@ type bitableFieldsResponse struct { // page_size here is rejected, so fields must be fetched 100 at a time and paged. const maxBitableFieldPageSize = 100 -type bitableRecordsResponse struct { - apiResponse - Data struct { - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - Items []struct { - Fields map[string]any `json:"fields"` - } `json:"items"` - } `json:"data"` +// bitableRecord is one entry of bitableRecordsData.Items. +type bitableRecord struct { + Fields map[string]any `json:"fields"` } -// ReadBitableRecords reads an embedded bitable block as a table: a header row of +// bitableRecordsData is the data payload of bitableRecordsResponse. +type bitableRecordsData struct { + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` + Items []bitableRecord `json:"items"` +} + +type bitableRecordsResponse struct { + ApiResponse + Data bitableRecordsData `json:"data"` +} + +// readBitableRecords reads an embedded bitable block as a table: a header row of // field names followed by one row per record. embedToken is the block's // bitable.token, formatted "appToken_tableId". Record rows are capped at // maxTableRows; truncated is true when the source had more records than that. -func (c *Client) ReadBitableRecords(ctx context.Context, embedToken string) ([][]string, bool, error) { +func (c *Client) readBitableRecords(ctx context.Context, embedToken string) ([][]string, bool, error) { idx := strings.LastIndex(embedToken, "_") if idx < 0 { return nil, false, fmt.Errorf("invalid bitable embed token: %q", embedToken) @@ -346,7 +382,7 @@ func (c *Client) ReadBitableRecords(ctx context.Context, embedToken string) ([][ fpath += "&page_token=" + url.QueryEscape(fieldPageToken) } var fieldsResp bitableFieldsResponse - if err := c.doRequest(ctx, http.MethodGet, fpath, nil, &fieldsResp); err != nil { + if err := c.DoRequest(ctx, http.MethodGet, fpath, nil, &fieldsResp); err != nil { return nil, false, fmt.Errorf("read bitable fields: %w", err) } if fieldsResp.Code != 0 { @@ -392,7 +428,7 @@ func (c *Client) ReadBitableRecords(ctx context.Context, embedToken string) ([][ rpath += "&page_token=" + url.QueryEscape(pageToken) } var rec bitableRecordsResponse - if err := c.doRequest(ctx, http.MethodPost, rpath, map[string]any{}, &rec); err != nil { + if err := c.DoRequest(ctx, http.MethodPost, rpath, map[string]any{}, &rec); err != nil { return nil, false, fmt.Errorf("search bitable records: %w", err) } if rec.Code != 0 { diff --git a/internal/datasource/connector/feishu/blocks_test.go b/internal/datasource/connector/feishu/core/blocks_test.go similarity index 92% rename from internal/datasource/connector/feishu/blocks_test.go rename to internal/datasource/connector/feishu/core/blocks_test.go index d76e6f032..00611814a 100644 --- a/internal/datasource/connector/feishu/blocks_test.go +++ b/internal/datasource/connector/feishu/core/blocks_test.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "context" @@ -38,9 +38,9 @@ func TestListDocumentBlocks_Paginates(t *testing.T) { defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - blocks, err := c.ListDocumentBlocks(context.Background(), "doc123") + blocks, err := c.listDocumentBlocks(context.Background(), "doc123") if err != nil { - t.Fatalf("ListDocumentBlocks: %v", err) + t.Fatalf("listDocumentBlocks: %v", err) } if len(blocks) != 3 { t.Fatalf("got %d blocks, want 3", len(blocks)) @@ -68,9 +68,9 @@ func TestReadSheetRange_SplitsTokenAndReadsValues(t *testing.T) { defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - rows, truncated, err := c.ReadSheetRange(context.Background(), "sht_abc_0") + rows, truncated, err := c.readSheetRange(context.Background(), "sht_abc_0") if err != nil { - t.Fatalf("ReadSheetRange: %v", err) + t.Fatalf("readSheetRange: %v", err) } if gotPath != "/open-apis/sheets/v2/spreadsheets/sht_abc/values/0" { t.Errorf("path = %q", gotPath) @@ -99,7 +99,7 @@ func TestReadSheetRange_TruncatesLargeTable(t *testing.T) { })) defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - rows, truncated, err := c.ReadSheetRange(context.Background(), "sht_x_0") + rows, truncated, err := c.readSheetRange(context.Background(), "sht_x_0") if err != nil { t.Fatalf("err: %v", err) } @@ -209,20 +209,22 @@ func TestReadBitableRecords_DateColumnUsesFormatterAndTimezone(t *testing.T) { "items": []map[string]any{ {"field_name": "截止日期", "type": 5, "property": map[string]any{"date_formatter": "yyyy/MM/dd"}}, {"field_name": "提醒时间", "type": 5, "property": map[string]any{"date_formatter": "yyyy-MM-dd HH:mm"}}, - }}}) + }, + }}) case strings.HasSuffix(r.URL.Path, "/records/search"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ "has_more": false, "items": []map[string]any{ {"fields": map[string]any{"截止日期": float64(1711900800000), "提醒时间": float64(1719802800000)}}, - }}}) + }, + }}) } })) defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client(), location: resolveLocation("")} - rows, _, err := c.ReadBitableRecords(context.Background(), "bascabc_tblxyz") + rows, _, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz") if err != nil { - t.Fatalf("ReadBitableRecords: %v", err) + t.Fatalf("readBitableRecords: %v", err) } if len(rows) != 2 { t.Fatalf("rows = %+v, want header + 1 record", rows) @@ -242,7 +244,8 @@ func TestReadBitableRecords_SplitsTokenAndBuildsTable(t *testing.T) { _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200}) case strings.HasSuffix(r.URL.Path, "/fields"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "items": []map[string]any{{"field_name": "任务"}, {"field_name": "状态"}}}}) + "items": []map[string]any{{"field_name": "任务"}, {"field_name": "状态"}}, + }}) case strings.HasSuffix(r.URL.Path, "/records/search"): // Must use the current Search-records endpoint (POST), not the // deprecated GET .../records legacy interface. @@ -256,15 +259,16 @@ func TestReadBitableRecords_SplitsTokenAndBuildsTable(t *testing.T) { "has_more": false, "items": []map[string]any{ {"fields": map[string]any{"任务": "写文档", "状态": "进行中"}}, - }}}) + }, + }}) } })) defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - rows, truncated, err := c.ReadBitableRecords(context.Background(), "bascabc_tblxyz") + rows, truncated, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz") if err != nil { - t.Fatalf("ReadBitableRecords: %v", err) + t.Fatalf("readBitableRecords: %v", err) } if truncated { t.Errorf("did not expect truncation") @@ -305,18 +309,20 @@ func TestReadBitableRecords_PaginatesFieldsAtMax100(t *testing.T) { } } _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "items": items, "has_more": hasMore, "page_token": nextTok}}) + "items": items, "has_more": hasMore, "page_token": nextTok, + }}) case strings.HasSuffix(r.URL.Path, "/records/search"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": false, "items": []map[string]any{}}}) + "has_more": false, "items": []map[string]any{}, + }}) } })) defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - rows, _, err := c.ReadBitableRecords(context.Background(), "bascabc_tblxyz") + rows, _, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz") if err != nil { - t.Fatalf("ReadBitableRecords: %v", err) + t.Fatalf("readBitableRecords: %v", err) } if len(rows) < 1 { t.Fatal("expected at least a header row") @@ -352,26 +358,29 @@ func TestReadBitableRecords_PaginatesAndTruncatesRecords(t *testing.T) { _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200}) case strings.HasSuffix(r.URL.Path, "/fields"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "items": []map[string]any{{"field_name": "col"}}}}) + "items": []map[string]any{{"field_name": "col"}}, + }}) case strings.HasSuffix(r.URL.Path, "/records/search"): if tok := r.URL.Query().Get("page_token"); tok == "" { // Page 1: 300 records, more to come. _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": true, "page_token": "r2", "items": bitableRows(300)}}) + "has_more": true, "page_token": "r2", "items": bitableRows(300), + }}) } else { secondPageToken = tok // Page 2: 250 more → 550 total, capped to maxTableRows. _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": false, "items": bitableRows(250)}}) + "has_more": false, "items": bitableRows(250), + }}) } } })) defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - rows, truncated, err := c.ReadBitableRecords(context.Background(), "bascabc_tblxyz") + rows, truncated, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz") if err != nil { - t.Fatalf("ReadBitableRecords: %v", err) + t.Fatalf("readBitableRecords: %v", err) } if secondPageToken != "r2" { t.Errorf("second records page must be fetched with page_token=r2, got %q", secondPageToken) @@ -393,18 +402,20 @@ func TestReadBitableRecords_Exactly500NotTruncated(t *testing.T) { _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200}) case strings.HasSuffix(r.URL.Path, "/fields"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "items": []map[string]any{{"field_name": "col"}}}}) + "items": []map[string]any{{"field_name": "col"}}, + }}) case strings.HasSuffix(r.URL.Path, "/records/search"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": false, "items": bitableRows(maxTableRows)}}) + "has_more": false, "items": bitableRows(maxTableRows), + }}) } })) defer srv.Close() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - rows, truncated, err := c.ReadBitableRecords(context.Background(), "bascabc_tblxyz") + rows, truncated, err := c.readBitableRecords(context.Background(), "bascabc_tblxyz") if err != nil { - t.Fatalf("ReadBitableRecords: %v", err) + t.Fatalf("readBitableRecords: %v", err) } if truncated { t.Error("exactly maxTableRows records must not be flagged truncated") @@ -427,11 +438,13 @@ func TestReadBitableRecords_EmptyRecordPageTerminates(t *testing.T) { _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "tenant_access_token": "t", "expire": 7200}) case strings.HasSuffix(r.URL.Path, "/fields"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "items": []map[string]any{{"field_name": "col"}}}}) + "items": []map[string]any{{"field_name": "col"}}, + }}) case strings.HasSuffix(r.URL.Path, "/records/search"): recordCalls++ _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": true, "page_token": "loop", "items": []map[string]any{}}}) + "has_more": true, "page_token": "loop", "items": []map[string]any{}, + }}) } })) defer srv.Close() @@ -439,7 +452,7 @@ func TestReadBitableRecords_EmptyRecordPageTerminates(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - rows, _, err := c.ReadBitableRecords(ctx, "bascabc_tblxyz") + rows, _, err := c.readBitableRecords(ctx, "bascabc_tblxyz") if err != nil { t.Fatalf("empty-page pagination must terminate cleanly, got err: %v", err) } @@ -462,10 +475,12 @@ func TestReadBitableRecords_EmptyFieldsPageTerminates(t *testing.T) { case strings.HasSuffix(r.URL.Path, "/fields"): fieldCalls++ _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": true, "page_token": "loop", "items": []map[string]any{}}}) + "has_more": true, "page_token": "loop", "items": []map[string]any{}, + }}) case strings.HasSuffix(r.URL.Path, "/records/search"): _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": false, "items": []map[string]any{}}}) + "has_more": false, "items": []map[string]any{}, + }}) } })) defer srv.Close() @@ -473,7 +488,7 @@ func TestReadBitableRecords_EmptyFieldsPageTerminates(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - if _, _, err := c.ReadBitableRecords(ctx, "bascabc_tblxyz"); err != nil { + if _, _, err := c.readBitableRecords(ctx, "bascabc_tblxyz"); err != nil { t.Fatalf("empty fields page must terminate cleanly, got err: %v", err) } if fieldCalls != 1 { @@ -491,7 +506,8 @@ func TestListDocumentBlocks_EmptyPageTerminates(t *testing.T) { case strings.HasSuffix(r.URL.Path, "/blocks"): blockCalls++ _ = json.NewEncoder(w).Encode(map[string]any{"code": 0, "data": map[string]any{ - "has_more": true, "page_token": "loop", "items": []map[string]any{}}}) + "has_more": true, "page_token": "loop", "items": []map[string]any{}, + }}) } })) defer srv.Close() @@ -499,7 +515,7 @@ func TestListDocumentBlocks_EmptyPageTerminates(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() c := &Client{baseURL: srv.URL, appID: "a", appSecret: "s", httpClient: srv.Client()} - blocks, err := c.ListDocumentBlocks(ctx, "doc-1") + blocks, err := c.listDocumentBlocks(ctx, "doc-1") if err != nil { t.Fatalf("empty blocks page must terminate cleanly, got err: %v", err) } diff --git a/internal/datasource/connector/feishu/client.go b/internal/datasource/connector/feishu/core/client.go similarity index 69% rename from internal/datasource/connector/feishu/client.go rename to internal/datasource/connector/feishu/core/client.go index 9c3da9bcd..52d12e3f7 100644 --- a/internal/datasource/connector/feishu/client.go +++ b/internal/datasource/connector/feishu/core/client.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "bytes" @@ -34,16 +34,16 @@ type Client struct { tokenExpAt time.Time } -type wikiNodeListFailure struct { - Node wikiNode +type WikiNodeListFailure struct { + Node WikiNode Err error } -type partialWikiNodeListError struct { - Failures []wikiNodeListFailure +type PartialWikiNodeListError struct { + Failures []WikiNodeListFailure } -func (e *partialWikiNodeListError) Error() string { +func (e *PartialWikiNodeListError) Error() string { if e == nil || len(e.Failures) == 0 { return "partial wiki node listing failed" } @@ -74,9 +74,9 @@ func NewClient(config *Config) *Client { } } -// getTenantAccessToken retrieves (or returns cached) tenant access token. +// GetTenantAccessToken retrieves (or returns cached) tenant access token. // Feishu tokens expire in 2 hours; we cache with a 5-minute safety margin. -func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) { +func (c *Client) GetTenantAccessToken(ctx context.Context) (string, error) { c.tokenMu.Lock() defer c.tokenMu.Unlock() @@ -102,7 +102,7 @@ func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) { } defer resp.Body.Close() - var result tokenResponse + var result TokenResponse if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { return "", fmt.Errorf("decode token response: %w", err) } @@ -131,7 +131,7 @@ func (c *Client) getTenantAccessToken(ctx context.Context) (string, error) { return c.tokenCache, nil } -// Retry policy shared by doRequest (JSON API calls) and downloadRawBytes (file +// Retry policy shared by DoRequest (JSON API calls) and downloadRawBytes (file // downloads): 429 honours Retry-After, 5xx retries once, transport errors back off. const ( feishuMaxRetries = 3 @@ -145,13 +145,13 @@ const maxFeishuDownloadBytes = 512 * 1024 * 1024 // 512 MB var feishuRetryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second} -// doRequest executes an authenticated API request and decodes the JSON response, +// DoRequest executes an authenticated API request and decodes the JSON response, // retrying transient failures (transport errors, HTTP 429, 5xx). Feishu's drive // export/wiki APIs are aggressively rate limited, and a thousand-document sync // issues tens of thousands of calls; without backoff a single 429 burst used to // fail whole swathes of documents silently. 429 responses honour Retry-After; // 5xx is retried once; other non-2xx statuses fail fast (no point retrying 4xx). -func (c *Client) doRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error { +func (c *Client) DoRequest(ctx context.Context, method, path string, body interface{}, result interface{}) error { const ( maxRetries = feishuMaxRetries max5xxRetries = feishuMax5xxRetries @@ -159,7 +159,7 @@ func (c *Client) doRequest(ctx context.Context, method, path string, body interf ) backoff := feishuRetryBackoff - token, err := c.getTenantAccessToken(ctx) + token, err := c.GetTenantAccessToken(ctx) if err != nil { return err } @@ -301,8 +301,8 @@ func truncate(s string, maxLen int) string { } // ListWikiSpaces returns all wiki spaces accessible to the app. -func (c *Client) ListWikiSpaces(ctx context.Context) ([]wikiSpace, error) { - var allSpaces []wikiSpace +func (c *Client) ListWikiSpaces(ctx context.Context) ([]WikiSpace, error) { + var allSpaces []WikiSpace pageToken := "" for { @@ -311,8 +311,8 @@ func (c *Client) ListWikiSpaces(ctx context.Context) ([]wikiSpace, error) { path += "&page_token=" + pageToken } - var resp wikiSpaceListResponse - if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + var resp WikiSpaceListResponse + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { return nil, fmt.Errorf("list wiki spaces: %w", err) } if resp.Code != 0 { @@ -339,8 +339,8 @@ func (c *Client) ListWikiSpaces(ctx context.Context) ([]wikiSpace, error) { // ListWikiNodes returns all nodes (documents) under a wiki space. // If parentNodeToken is empty, returns top-level nodes. -func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeToken string) ([]wikiNode, error) { - var allNodes []wikiNode +func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeToken string) ([]WikiNode, error) { + var allNodes []WikiNode pageToken := "" for { @@ -352,8 +352,8 @@ func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeTo path += "&page_token=" + pageToken } - var resp wikiNodeListResponse - if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + var resp WikiNodeListResponse + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { return nil, fmt.Errorf("list wiki nodes: %w", err) } if resp.Code != 0 { @@ -380,15 +380,15 @@ func (c *Client) ListWikiNodes(ctx context.Context, spaceID string, parentNodeTo } // GetWikiNode returns metadata for a single wiki node. -func (c *Client) GetWikiNode(ctx context.Context, spaceID string, nodeToken string) (wikiNode, error) { +func (c *Client) GetWikiNode(ctx context.Context, spaceID string, nodeToken string) (WikiNode, error) { path := fmt.Sprintf("/open-apis/wiki/v2/spaces/get_node?token=%s", url.QueryEscape(nodeToken)) - var resp wikiNodeInfoResponse - if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { - return wikiNode{}, fmt.Errorf("get wiki node: %w", err) + var resp WikiNodeInfoResponse + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + return WikiNode{}, fmt.Errorf("get wiki node: %w", err) } if resp.Code != 0 { - return wikiNode{}, fmt.Errorf("get wiki node error: code=%d msg=%s", resp.Code, resp.Msg) + return WikiNode{}, fmt.Errorf("get wiki node error: code=%d msg=%s", resp.Code, resp.Msg) } node := resp.Data.Node @@ -398,20 +398,20 @@ func (c *Client) GetWikiNode(ctx context.Context, spaceID string, nodeToken stri return node, nil } -// ListAllWikiNodesRecursive recursively lists all nodes under a wiki space. +// listAllWikiNodesRecursive recursively lists all nodes under a wiki space. // It walks the tree depth-first to discover all nested documents. -func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) ([]wikiNode, error) { +func (c *Client) listAllWikiNodesRecursive(ctx context.Context, spaceID string) ([]WikiNode, error) { // Start with top-level nodes topNodes, err := c.ListWikiNodes(ctx, spaceID, "") if err != nil { return nil, err } - var allNodes []wikiNode - var failures []wikiNodeListFailure - var walk func(nodes []wikiNode) + var allNodes []WikiNode + var failures []WikiNodeListFailure + var walk func(nodes []WikiNode) - walk = func(nodes []wikiNode) { + walk = func(nodes []WikiNode) { for _, node := range nodes { allNodes = append(allNodes, node) @@ -420,7 +420,7 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) children, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken) if err != nil { wrappedErr := fmt.Errorf("list children of %s: %w", node.NodeToken, err) - failures = append(failures, wikiNodeListFailure{ + failures = append(failures, WikiNodeListFailure{ Node: node, Err: wrappedErr, }) @@ -435,16 +435,16 @@ func (c *Client) ListAllWikiNodesRecursive(ctx context.Context, spaceID string) walk(topNodes) if len(failures) > 0 { - return allNodes, &partialWikiNodeListError{Failures: failures} + return allNodes, &PartialWikiNodeListError{Failures: failures} } return allNodes, nil } // ListWikiNodesRecursiveFrom returns a wiki node and all descendants below it. -func (c *Client) ListWikiNodesRecursiveFrom(ctx context.Context, spaceID string, nodeToken string) ([]wikiNode, error) { +func (c *Client) ListWikiNodesRecursiveFrom(ctx context.Context, spaceID string, nodeToken string) ([]WikiNode, error) { if nodeToken == "" { - return c.ListAllWikiNodesRecursive(ctx, spaceID) + return c.listAllWikiNodesRecursive(ctx, spaceID) } root, err := c.GetWikiNode(ctx, spaceID, nodeToken) @@ -454,12 +454,12 @@ func (c *Client) ListWikiNodesRecursiveFrom(ctx context.Context, spaceID string, nodes, err := c.listWikiNodeDescendants(ctx, spaceID, root) if err != nil { - return append([]wikiNode{root}, nodes...), err + return append([]WikiNode{root}, nodes...), err } - return append([]wikiNode{root}, nodes...), nil + return append([]WikiNode{root}, nodes...), nil } -func (c *Client) listWikiNodeDescendants(ctx context.Context, spaceID string, root wikiNode) ([]wikiNode, error) { +func (c *Client) listWikiNodeDescendants(ctx context.Context, spaceID string, root WikiNode) ([]WikiNode, error) { if !root.HasChild { return nil, nil } @@ -469,19 +469,19 @@ func (c *Client) listWikiNodeDescendants(ctx context.Context, spaceID string, ro wrappedErr := fmt.Errorf("list children of %s: %w", root.NodeToken, err) logger.Warnf(ctx, "[Feishu] partial wiki node listing failure: space=%s node=%s err=%v", spaceID, root.NodeToken, err) - return nil, &partialWikiNodeListError{ - Failures: []wikiNodeListFailure{{ + return nil, &PartialWikiNodeListError{ + Failures: []WikiNodeListFailure{{ Node: root, Err: wrappedErr, }}, } } - var allNodes []wikiNode - var failures []wikiNodeListFailure - var walk func(nodes []wikiNode) + var allNodes []WikiNode + var failures []WikiNodeListFailure + var walk func(nodes []WikiNode) - walk = func(nodes []wikiNode) { + walk = func(nodes []WikiNode) { for _, node := range nodes { allNodes = append(allNodes, node) if !node.HasChild { @@ -491,7 +491,7 @@ func (c *Client) listWikiNodeDescendants(ctx context.Context, spaceID string, ro grandChildren, err := c.ListWikiNodes(ctx, spaceID, node.NodeToken) if err != nil { wrappedErr := fmt.Errorf("list children of %s: %w", node.NodeToken, err) - failures = append(failures, wikiNodeListFailure{ + failures = append(failures, WikiNodeListFailure{ Node: node, Err: wrappedErr, }) @@ -505,19 +505,19 @@ func (c *Client) listWikiNodeDescendants(ctx context.Context, spaceID string, ro walk(children) if len(failures) > 0 { - return allNodes, &partialWikiNodeListError{Failures: failures} + return allNodes, &PartialWikiNodeListError{Failures: failures} } return allNodes, nil } -// GetDocumentRawContent retrieves the raw text content of a Feishu docx document. +// getDocumentRawContent retrieves the raw text content of a Feishu docx document. // This returns plain text (not rich text / block structure). // Deprecated: prefer ExportAndDownload which preserves formatting. -func (c *Client) GetDocumentRawContent(ctx context.Context, documentID string) (string, error) { +func (c *Client) getDocumentRawContent(ctx context.Context, documentID string) (string, error) { path := fmt.Sprintf("/open-apis/docx/v1/documents/%s/raw_content", documentID) var resp docRawContentResponse - if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { return "", fmt.Errorf("get document raw content: %w", err) } if resp.Code != 0 { @@ -529,7 +529,7 @@ func (c *Client) GetDocumentRawContent(ctx context.Context, documentID string) ( // Ping verifies the credentials by attempting to get a tenant access token. func (c *Client) Ping(ctx context.Context) error { - _, err := c.getTenantAccessToken(ctx) + _, err := c.GetTenantAccessToken(ctx) return err } @@ -542,19 +542,19 @@ func (c *Client) Ping(ctx context.Context) error { // 3. GET /drive/v1/export_tasks/file/:ticket/download → download file bytes // ────────────────────────────────────────────────────────────────────── -// CreateExportTask creates an async export task for a Feishu document. +// createExportTask creates an async export task for a Feishu document. // - token: the obj_token of the document (e.g. docx token, sheet token) // - objType: the Feishu obj_type ("docx", "doc", "sheet", "bitable") // - fileExtension: desired output format ("docx", "xlsx", "pdf") -func (c *Client) CreateExportTask(ctx context.Context, token, objType, fileExtension string) (string, error) { +func (c *Client) createExportTask(ctx context.Context, token, objType, fileExtension string) (string, error) { body := map[string]string{ "file_extension": fileExtension, "token": token, "type": objType, } - var resp exportTaskCreateResponse - if err := c.doRequest(ctx, http.MethodPost, "/open-apis/drive/v1/export_tasks", body, &resp); err != nil { + var resp ExportTaskCreateResponse + if err := c.DoRequest(ctx, http.MethodPost, "/open-apis/drive/v1/export_tasks", body, &resp); err != nil { return "", fmt.Errorf("create export task: %w", err) } if resp.Code != 0 { @@ -564,14 +564,14 @@ func (c *Client) CreateExportTask(ctx context.Context, token, objType, fileExten return resp.Data.Ticket, nil } -// GetExportTaskStatus polls the status of an export task. +// getExportTaskStatus polls the status of an export task. // Returns (fileToken, fileName, error). fileToken is non-empty only when the job succeeds. // The token parameter is the obj_token of the document being exported (required by the API). -func (c *Client) GetExportTaskStatus(ctx context.Context, ticket string, token string) (string, string, error) { +func (c *Client) getExportTaskStatus(ctx context.Context, ticket string, token string) (string, string, error) { path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/%s?token=%s", ticket, token) - var resp exportTaskStatusResponse - if err := c.doRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + var resp ExportTaskStatusResponse + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { return "", "", fmt.Errorf("get export task status: %w", err) } if resp.Code != 0 { @@ -589,10 +589,10 @@ func (c *Client) GetExportTaskStatus(ctx context.Context, ticket string, token s } } -// DownloadExportFile downloads the exported file by its file_token. -// The file_token is returned by GetExportTaskStatus when the export job completes. +// downloadExportFile downloads the exported file by its file_token. +// The file_token is returned by getExportTaskStatus when the export job completes. // The file must be downloaded within 10 minutes of export completion. -func (c *Client) DownloadExportFile(ctx context.Context, fileToken string) ([]byte, error) { +func (c *Client) downloadExportFile(ctx context.Context, fileToken string) ([]byte, error) { path := fmt.Sprintf("/open-apis/drive/v1/export_tasks/file/%s/download", fileToken) return c.downloadRawBytes(ctx, path) } @@ -603,18 +603,18 @@ func (c *Client) DownloadExportFile(ctx context.Context, fileToken string) ([]by // Timeout: 60 seconds. Poll interval: 2 seconds. func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string) ([]byte, string, error) { // Determine export format - fileExt, ok := objTypeToExportFileExtension[objType] + fileExt, ok := ObjTypeToExportFileExtension[objType] if !ok { return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType) } - exportType, ok := objTypeToExportType[objType] + exportType, ok := ObjTypeToExportType[objType] if !ok { return nil, "", fmt.Errorf("unsupported obj_type for export: %s", objType) } // Step 1: create export task - ticket, err := c.CreateExportTask(ctx, objToken, exportType, fileExt) + ticket, err := c.createExportTask(ctx, objToken, exportType, fileExt) if err != nil { return nil, "", err } @@ -624,7 +624,7 @@ func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string var fileToken, fileName string for time.Now().Before(deadline) { - fileToken, fileName, err = c.GetExportTaskStatus(ctx, ticket, objToken) + fileToken, fileName, err = c.getExportTaskStatus(ctx, ticket, objToken) if err != nil { return nil, "", err } @@ -643,14 +643,14 @@ func (c *Client) ExportAndDownload(ctx context.Context, objToken, objType string } // Step 3: download file using file_token (NOT ticket) - data, err := c.DownloadExportFile(ctx, fileToken) + data, err := c.downloadExportFile(ctx, fileToken) if err != nil { return nil, "", err } // Build a sensible file name if fileName == "" { - fileName = "export" + exportFileExtToSuffix[fileExt] + fileName = "export" + ExportFileExtToSuffix[fileExt] } return data, fileName, nil @@ -667,18 +667,18 @@ func (c *Client) DownloadDriveFile(ctx context.Context, fileToken string) ([]byt return c.downloadRawBytes(ctx, path) } -// DownloadMediaFile downloads embedded media (attachments/images referenced by +// downloadMediaFile downloads embedded media (attachments/images referenced by // document File/Image blocks) by its media token. Embedded block media live in a // different token space than standalone Drive files and must use the /medias/ // endpoint rather than /files/. -func (c *Client) DownloadMediaFile(ctx context.Context, fileToken string) ([]byte, error) { +func (c *Client) downloadMediaFile(ctx context.Context, fileToken string) ([]byte, error) { path := fmt.Sprintf("/open-apis/drive/v1/medias/%s/download", url.PathEscape(fileToken)) return c.downloadRawBytes(ctx, path) } // downloadRawBytes performs an authenticated GET and returns the raw response body. func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, error) { - token, err := c.getTenantAccessToken(ctx) + token, err := c.GetTenantAccessToken(ctx) if err != nil { return nil, err } @@ -768,3 +768,149 @@ func (c *Client) downloadRawBytes(ctx context.Context, path string) ([]byte, err return nil, lastErr } + +// ────────────────────────────────────────────────────────────────────── +// Drive (云盘) file listing: for feishu_drive / lark_drive connectors. +// Mirrors the wiki ListWikiNodes / ListWikiNodesRecursiveFrom shape so the +// Drive connector's FetchStream mirrors the wiki connector's. See ADR-0001/0002. +// ────────────────────────────────────────────────────────────────────── + +// listDriveFiles lists files in a Drive folder (non-recursive), one page at a +// time. Pass pageToken="" for the first page; the returned nextPageToken is "" +// when there are no more pages. +// +// folderToken == "" is rejected: the root folder is not paginated and does +// not return shortcuts (Feishu API limitation), which would silently drop +// content and risk an unbounded single response. See ADR-0004. +func (c *Client) listDriveFiles(ctx context.Context, folderToken, pageToken string) ([]DriveFile, string, error) { + if folderToken == "" { + return nil, "", fmt.Errorf("root folder not supported; specify a concrete folder_token (root folder is not paginated and does not return shortcuts)") + } + + path := "/open-apis/drive/v1/files?folder_token=" + url.QueryEscape(folderToken) + path += "&page_size=200" // max + path += "&order_by=EditedTime&direction=DESC" + if pageToken != "" { + path += "&page_token=" + url.QueryEscape(pageToken) + } + + var resp DriveFileListResponse + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + return nil, "", fmt.Errorf("list drive files: %w", err) + } + if resp.Code != 0 { + return nil, "", fmt.Errorf("list drive files error: code=%d msg=%s", resp.Code, resp.Msg) + } + + logger.Infof(ctx, "[FeishuDrive] listDriveFiles: folder=%s got %d files, has_more=%v", + folderToken, len(resp.Data.Files), resp.Data.HasMore) + return resp.Data.Files, resp.Data.NextPageToken, nil +} + +// GetDriveFolderMeta returns the metadata (name, owner, etc.) of a single Drive +// folder. Used to resolve a root folder's human-readable name - the list API +// only returns the folder's children, not the folder itself. +// +// GET /open-apis/drive/explorer/v2/folder/:folderToken/meta +func (c *Client) GetDriveFolderMeta(ctx context.Context, folderToken string) (driveFolderMetaResponse, error) { + var resp driveFolderMetaResponse + if folderToken == "" { + return resp, fmt.Errorf("root folder not supported; specify a concrete folder_token") + } + path := "/open-apis/drive/explorer/v2/folder/" + url.QueryEscape(folderToken) + "/meta" + if err := c.DoRequest(ctx, http.MethodGet, path, nil, &resp); err != nil { + return resp, fmt.Errorf("get drive folder meta: %w", err) + } + if resp.Code != 0 { + return resp, fmt.Errorf("get drive folder meta error: code=%d msg=%s", resp.Code, resp.Msg) + } + return resp, nil +} + +// ListDriveFilesAllPages lists every direct child of a folder across all pages. +func (c *Client) ListDriveFilesAllPages(ctx context.Context, folderToken string) ([]DriveFile, error) { + var all []DriveFile + pageToken := "" + for { + files, next, err := c.listDriveFiles(ctx, folderToken, pageToken) + if err != nil { + return nil, err + } + all = append(all, files...) + if next == "" { + break + } + pageToken = next + } + return all, nil +} + +// ListDriveFilesRecursiveFrom walks a Drive folder subtree depth-first, +// returning all non-folder files. Mirrors ListWikiNodesRecursiveFrom. +// +// - folder -> recurse (visited is a pure-defensive cycle guard; Drive folders +// have no shortcut concept so cycles are not expected - see glossary). +// - shortcut -> expand to its target (target_type is never "folder", verified) +// and include the target as a regular file. No extra API call: shortcut_info +// is returned by the list API. +// - other -> collect. +// +// Partial failures (a sub-folder listing returns an error) are collected into a +// *PartialDriveFileListError and the walk continues, mirroring the wiki +// connector's PartialWikiNodeListError semantics. +func (c *Client) ListDriveFilesRecursiveFrom(ctx context.Context, folderToken string) ([]DriveFile, error) { + visited := make(map[string]bool) + var all []DriveFile + var failures []DriveFileListFailure + + var walk func(folderToken string) + walk = func(folderToken string) { + if visited[folderToken] { + return + } + visited[folderToken] = true + + files, err := c.ListDriveFilesAllPages(ctx, folderToken) + if err != nil { + wrappedErr := fmt.Errorf("list children of %s: %w", folderToken, err) + failures = append(failures, DriveFileListFailure{ + FolderToken: folderToken, + Err: wrappedErr, + }) + logger.Warnf(ctx, "[FeishuDrive] partial drive file listing failure: folder=%s err=%v", + folderToken, err) + return + } + + for _, f := range files { + switch f.Type { + case "folder": + walk(f.Token) + case "shortcut": + // Expand to target. target_type is never "folder" (verified), so + // no recursion here - the target is a regular file. + if f.ShortcutInfo != nil && f.ShortcutInfo.TargetToken != "" { + expanded := DriveFile{ + Token: f.ShortcutInfo.TargetToken, + Name: f.Name, + Type: f.ShortcutInfo.TargetType, + ParentToken: f.ParentToken, + URL: f.URL, + CreatedTime: f.CreatedTime, + ModifiedTime: f.ModifiedTime, + OwnerID: f.OwnerID, + } + all = append(all, expanded) + } + default: + all = append(all, f) + } + } + } + + walk(folderToken) + if len(failures) > 0 { + return all, &PartialDriveFileListError{Failures: failures} + } + return all, nil +} diff --git a/internal/datasource/connector/feishu/client_retry_test.go b/internal/datasource/connector/feishu/core/client_retry_test.go similarity index 90% rename from internal/datasource/connector/feishu/client_retry_test.go rename to internal/datasource/connector/feishu/core/client_retry_test.go index 0120ebf5f..0e9444918 100644 --- a/internal/datasource/connector/feishu/client_retry_test.go +++ b/internal/datasource/connector/feishu/core/client_retry_test.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "context" @@ -10,12 +10,12 @@ import ( ) // retryTestServer builds a server that always answers the auth-token call and -// routes the given target path to h, so tests can drive doRequest's retry loop. +// routes the given target path to h, so tests can drive DoRequest's retry loop. func retryTestServer(target string, h http.HandlerFunc) (*httptest.Server, *Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{ - apiResponse: apiResponse{Code: 0}, + writeJSON(w, TokenResponse{ + ApiResponse: ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) @@ -36,13 +36,13 @@ func TestDoRequest_RetriesOn429ThenSucceeds(t *testing.T) { _, _ = io.WriteString(w, `{"code":99991400,"msg":"rate limited"}`) return } - writeJSON(w, apiResponse{Code: 0}) + writeJSON(w, ApiResponse{Code: 0}) }) defer ts.Close() c := NewClient(cfg) - var resp apiResponse - if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil { + var resp ApiResponse + if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, &resp); err != nil { t.Fatalf("expected success after retry, got %v", err) } if attempts < 2 { @@ -61,7 +61,7 @@ func TestDoRequest_429ExhaustsRetries(t *testing.T) { defer ts.Close() c := NewClient(cfg) - err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil) + err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil) if err == nil { t.Fatal("expected error when 429s exceed the retry budget") } @@ -83,7 +83,7 @@ func TestDoRequest_5xxRetriesOnce(t *testing.T) { defer cancel() c := NewClient(cfg) - if err := c.doRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil { + if err := c.DoRequest(ctx, http.MethodGet, "/target", nil, nil); err == nil { t.Fatal("expected error after 5xx exhaustion") } if attempts != 2 { // initial + 1 retry @@ -101,7 +101,7 @@ func TestDoRequest_4xxNotRetried(t *testing.T) { defer ts.Close() c := NewClient(cfg) - if err := c.doRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil { + if err := c.DoRequest(context.Background(), http.MethodGet, "/target", nil, nil); err == nil { t.Fatal("expected error on 400") } if attempts != 1 { diff --git a/internal/datasource/connector/feishu/connector_error_reason_test.go b/internal/datasource/connector/feishu/core/connector_error_reason_test.go similarity index 97% rename from internal/datasource/connector/feishu/connector_error_reason_test.go rename to internal/datasource/connector/feishu/core/connector_error_reason_test.go index 7563a67d4..ffec83dce 100644 --- a/internal/datasource/connector/feishu/connector_error_reason_test.go +++ b/internal/datasource/connector/feishu/core/connector_error_reason_test.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "errors" @@ -40,7 +40,7 @@ func TestFeishuFailure(t *testing.T) { }, { name: "api error carries the feishu code as a param", - err: errors.New("feishu api error: status=500 body={\"code\":1663,\"msg\":\"internal error\",\"error\":{\"log_id\":\"20260\"}}"), + err: errors.New("feishu api error: status=500 body={\"code\":1663,\"msg\":\"internal error\",\"Error\":{\"log_id\":\"20260\"}}"), wantCode: "feishu_api_error", wantCodeValue: "1663", noLeak: []string{"log_id", "body=", "{"}, diff --git a/internal/datasource/connector/feishu/core/engine.go b/internal/datasource/connector/feishu/core/engine.go new file mode 100644 index 000000000..efdca0f8f --- /dev/null +++ b/internal/datasource/connector/feishu/core/engine.go @@ -0,0 +1,263 @@ +package core + +import ( + "context" + "fmt" + "time" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// engine.go holds the single generic streaming sync engine shared by the wiki +// Connector and the Drive DriveConnector. The per-connector differences (node +// type, listing API, edit-time field, cursor wire format, fetch dispatch, +// log tag) are isolated behind the NodeOps adapter interface. FetchAll / +// FetchIncremental are thin wrappers over the same engine that collect Emits +// instead of streaming them. +// +// Behaviour note (deliberate, see ADR-0005 / design §2.4): +// - Resume/incremental fast-path: a node recorded at its current edit time +// is skipped, keeping the cursor entry. +// - A fetch failure does NOT advance the cursor: the prior edit time is +// retained so the node is retried next run instead of being permanently +// skipped on a transient export failure (Tencent/WeKnora#2136). This now +// also holds for the FetchIncremental path (previously it advanced the +// cursor before fetching, a latent #2136 bug). +// - Logs use the "stream progress/summary" wording uniformly; the +// FetchIncremental path additionally gains per-100 progress + tally +// summary logs it did not Emit before (log-only change). + +// NodeOps adapts one connector's node type to the shared sync engine. Every +// method is a pure accessor or a thin wrapper - no engine logic lives here. +type NodeOps[N any] interface { + // List returns every syncable node under resourceID. A non-nil partial + // (with err == nil) signals a partial listing: nodes is still usable and + // the sync continues, but the caller surfaces the failed sub-trees via + // ListFailureItems. A non-nil err is fatal and aborts the sync. + List(ctx context.Context, client *Client, resourceID string) (nodes []N, partial error, err error) + + Token(n N) string + Title(n N) string + ObjType(n N) string + // EditTime is the change-detection timestamp string stored in the cursor. + EditTime(n N) string + + // fetch retrieves one node's content; (nil, nil) means an unsupported + // type that yields no item. + Fetch(ctx context.Context, client *Client, n N, resourceID string, multimodal bool) ([]*types.FetchedItem, error) + + // ListFailureItems converts a partial-listing error into error FetchedItems. + ListFailureItems(resourceID string, partial error) []types.FetchedItem + // ResourceNoun is the noun in user-visible error text: "nodes" / "files". + ResourceNoun() string + // EmptyResourceIDsError is the connector-specific message when no resource + // IDs are configured (wiki/drive text differs and is preserved verbatim). + EmptyResourceIDsError() string + // LogTag is the log prefix: "[Feishu]" / "[FeishuDrive]". + LogTag() string + + // DecodeCursorTimes extracts the per-resource edit-time map from a + // persisted ConnectorCursor (nil-safe: returns nil when absent). + DecodeCursorTimes(m map[string]interface{}) map[string]map[string]string + // EncodeCursor wraps the engine's internal times map into the connector's + // wire-format SyncCursor. JSON-marshals for snapshot isolation, mirroring + // the original FeishuCursor.toSyncCursor / FeishuDriveCursor.toSyncCursor. + EncodeCursor(times map[string]map[string]string, lastSync time.Time) *types.SyncCursor +} + +// CollectHandler is the StreamHandler used by FetchAll / FetchIncremental to +// gather every Emitted item into a slice instead of streaming. Checkpoint is a +// no-op: those paths return a single cursor at the end. +type CollectHandler struct { + items []types.FetchedItem +} + +func (h *CollectHandler) Emit(_ context.Context, item types.FetchedItem) error { + h.items = append(h.items, item) + return nil +} + +func (h *CollectHandler) Checkpoint(_ context.Context, _ *types.SyncCursor) error { return nil } + +// runSync is the single implementation behind FetchStream / FetchAll / +// FetchIncremental. With cursor == nil it fetches everything (full sync); +// with a cursor it skips nodes whose recorded edit time is unchanged +// (incremental + resume). resourceIDs comes from the caller: FetchStream / +// FetchIncremental pass config.ResourceIDs (after a non-empty check), +// FetchAll passes its own argument without one. +func runSync[N any]( + ctx context.Context, client *Client, config *types.DataSourceConfig, + resourceIDs []string, cursor *types.SyncCursor, h datasource.StreamHandler, + ops NodeOps[N], +) (*types.SyncCursor, error) { + var prevTimes map[string]map[string]string + if cursor != nil && cursor.ConnectorCursor != nil { + prevTimes = ops.DecodeCursorTimes(cursor.ConnectorCursor) + } + + newTimes := make(map[string]map[string]string) + lastSync := time.Now() + + processed := 0 + lastCheckpoint := time.Now() + for _, resourceID := range resourceIDs { + nodes, partial, err := ops.List(ctx, client, resourceID) + if err != nil { + return nil, fmt.Errorf("list %s for resource %s: %w", ops.ResourceNoun(), resourceID, err) + } + if partial != nil { + for _, item := range ops.ListFailureItems(resourceID, partial) { + if eerr := h.Emit(ctx, item); eerr != nil { + return nil, eerr + } + } + } + + newTimes[resourceID] = make(map[string]string) + // On a partial listing, carry prior edit times forward so a later full + // listing can still detect changes and deletions. + if partial != nil && prevTimes != nil { + if prev, ok := prevTimes[resourceID]; ok { + for tok, et := range prev { + newTimes[resourceID][tok] = et + } + } + } + + currentNodes := make(map[string]bool) + tally := newFetchTally(len(nodes)) + for i, node := range nodes { + tok := ops.Token(node) + currentNodes[tok] = true + editTimeStr := ops.EditTime(node) + + var prevEdit string + var hadPrev bool + if prevTimes != nil { + if prev, ok := prevTimes[resourceID]; ok { + prevEdit, hadPrev = prev[tok] + } + } + + // Resume/incremental fast-path: a node recorded at its current edit + // time is unchanged (or already synced this run) - keep the record + // and Skip re-fetching. + if hadPrev && prevEdit == editTimeStr { + newTimes[resourceID][tok] = editTimeStr + continue + } + + items, ferr := ops.Fetch(ctx, client, node, resourceID, config.MultimodalEnabled) + if ferr != nil { + tally.fail() + // Do NOT advance the cursor: the content was never fetched. + // Retain the prior edit time (if any) so prev != current next + // run and the node is retried, instead of being permanently + // skipped on a transient export failure (Tencent/WeKnora#2136). + if hadPrev { + newTimes[resourceID][tok] = prevEdit + } + if eerr := h.Emit(ctx, types.FetchedItem{ + ExternalID: tok, + Title: ops.Title(node), + SourceResourceID: resourceID, + Metadata: FeishuErrorItemMeta(ferr, nil), + }); eerr != nil { + return nil, eerr + } + } else { + // Fetched, or an unsupported type (nothing to fetch): record + // the current edit time so the node is not re-processed next run. + newTimes[resourceID][tok] = editTimeStr + if len(items) > 0 { + tally.fetch() + for _, it := range items { + if eerr := h.Emit(ctx, *it); eerr != nil { + return nil, eerr + } + } + } else { + // Unsupported type (mindnote/slides/…): no item. + tally.Skip(ops.ObjType(node)) + } + } + + processed++ + if processed%FeishuStreamCheckpointInterval == 0 || time.Since(lastCheckpoint) >= FeishuStreamCheckpointMaxInterval { + if cerr := h.Checkpoint(ctx, ops.EncodeCursor(newTimes, lastSync)); cerr != nil { + logger.Warnf(ctx, "%s stream Checkpoint failed: %v", ops.LogTag(), cerr) + } + lastCheckpoint = time.Now() + } + if n := i + 1; n%100 == 0 { + logger.Infof(ctx, "%s stream progress resource=%s %d/%d (%s)", + ops.LogTag(), resourceID, n, len(nodes), tally.summary()) + } + } + + // Detect deleted nodes (only when the full tree was listed successfully). + // A partial listing did not enumerate the whole subtree, so deletion + // detection would false-positive. + if partial == nil && prevTimes != nil { + if prev, ok := prevTimes[resourceID]; ok { + for tok := range prev { + if !currentNodes[tok] { + if eerr := h.Emit(ctx, types.FetchedItem{ + ExternalID: tok, + IsDeleted: true, + SourceResourceID: resourceID, + }); eerr != nil { + return nil, eerr + } + } + } + } + } + logger.Infof(ctx, "%s stream summary resource=%s %s", ops.LogTag(), resourceID, tally.summary()) + } + + return ops.EncodeCursor(newTimes, lastSync), nil +} + +// FetchStreamEngine runs the streaming sync. FetchStream / FetchIncremental +// shells differ only in whether they pass a cursor and how they collect +// results; both route here. +func FetchStreamEngine[N any]( + ctx context.Context, client *Client, config *types.DataSourceConfig, + cursor *types.SyncCursor, h datasource.StreamHandler, ops NodeOps[N], +) (*types.SyncCursor, error) { + return runSync(ctx, client, config, config.ResourceIDs, cursor, h, ops) +} + +// FetchAllEngine runs a full sync, collecting every item into a slice. It +// passes the caller-supplied resourceIDs without a non-empty check (mirroring +// the original FetchAll which accepted an empty list and returned no items). +// On error any partially collected items are discarded, matching the original +// behaviour (a fatal list error dropped everything collected so far). +func FetchAllEngine[N any]( + ctx context.Context, client *Client, config *types.DataSourceConfig, + resourceIDs []string, ops NodeOps[N], +) ([]types.FetchedItem, error) { + ch := &CollectHandler{} + if _, err := runSync(ctx, client, config, resourceIDs, nil, ch, ops); err != nil { + return nil, err + } + return ch.items, nil +} + +// FetchIncrementalEngine runs an incremental sync against cursor, collecting +// items. resourceIDs come from config.ResourceIDs (non-empty check is the +// caller's responsibility). +func FetchIncrementalEngine[N any]( + ctx context.Context, client *Client, config *types.DataSourceConfig, + cursor *types.SyncCursor, ops NodeOps[N], +) ([]types.FetchedItem, *types.SyncCursor, error) { + ch := &CollectHandler{} + next, err := runSync(ctx, client, config, config.ResourceIDs, cursor, ch, ops) + if err != nil { + return nil, nil, err + } + return ch.items, next, nil +} diff --git a/internal/datasource/connector/feishu/core/helpers_test.go b/internal/datasource/connector/feishu/core/helpers_test.go new file mode 100644 index 000000000..aa6051797 --- /dev/null +++ b/internal/datasource/connector/feishu/core/helpers_test.go @@ -0,0 +1,54 @@ +package core + +import ( + "encoding/json" + "net/http" + "os" + "testing" + + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +func TestMain(m *testing.M) { + os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost,open.feishu.cn,open.larksuite.com") + secutils.ResetSSRFWhitelistForTest() + os.Exit(m.Run()) +} + +func writeJSON(w http.ResponseWriter, v interface{}) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +// blk constructors mirroring wiki/connector_golden_test.go, for core-internal +// block-rendering tests (DocxBlock is same-package here, no core. prefix). +func sheetBlk(id, token string) DocxBlock { + return DocxBlock{BlockID: id, BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: token}} +} + +func bitableBlk(id, token string) DocxBlock { + return DocxBlock{BlockID: id, BlockType: BlockTypeBitable, Bitable: &BlockTokenRef{Token: token}} +} + +func imageBlk(id, token string) DocxBlock { + return DocxBlock{BlockID: id, BlockType: BlockTypeImage, Image: &BlockTokenRef{Token: token}} +} + +func fileBlk(id, token, name string) DocxBlock { + return DocxBlock{BlockID: id, BlockType: BlockTypeFile, File: &BlockFileRef{Token: token, Name: name}} +} + +func cellBlk(id string) DocxBlock { + return DocxBlock{BlockID: id, BlockType: BlockTypeTableCell, Children: []string{id + "_txt"}} +} + +func cellTextBlk(id, text string) DocxBlock { + return DocxBlock{BlockID: id + "_txt", BlockType: BlockTypeText, Text: txt(text)} +} + +func tableBlk(id string, cols int, cellIDs ...string) DocxBlock { + b := DocxBlock{BlockID: id, BlockType: BlockTypeTable} + b.Table = &BlockTable{Cells: cellIDs} + b.Table.Property = &BlockTableProperty{ColumnSize: cols} + return b +} diff --git a/internal/datasource/connector/feishu/markdown.go b/internal/datasource/connector/feishu/core/markdown.go similarity index 82% rename from internal/datasource/connector/feishu/markdown.go rename to internal/datasource/connector/feishu/core/markdown.go index 66964d5a3..faf395375 100644 --- a/internal/datasource/connector/feishu/markdown.go +++ b/internal/datasource/connector/feishu/core/markdown.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "context" @@ -9,8 +9,8 @@ import ( // sheetReader is the subset of *Client that blocksToMarkdown needs, so the // converter can be unit-tested with a fake or nil client. type sheetReader interface { - ReadSheetRange(ctx context.Context, embedToken string) ([][]string, bool, error) - ReadBitableRecords(ctx context.Context, embedToken string) ([][]string, bool, error) + readSheetRange(ctx context.Context, embedToken string) ([][]string, bool, error) + readBitableRecords(ctx context.Context, embedToken string) ([][]string, bool, error) } // pendingAttachment is an embedded file block awaiting a download decision by @@ -23,14 +23,14 @@ type pendingAttachment struct { // blocksToMarkdown renders a flat docx block array to Markdown, inlining // embedded spreadsheet/bitable tables (Task 5) and collecting downloadable // attachments. client may be nil when the block set has no downdrill blocks. -func blocksToMarkdown(ctx context.Context, client sheetReader, blocks []docxBlock) ([]byte, []pendingAttachment, error) { - byID := make(map[string]docxBlock, len(blocks)) +func blocksToMarkdown(ctx context.Context, client sheetReader, blocks []DocxBlock) ([]byte, []pendingAttachment, error) { + byID := make(map[string]DocxBlock, len(blocks)) for _, b := range blocks { byID[b.BlockID] = b } // Blocks nested inside a native table (its cells and their content blocks) // are rendered by renderNativeTable. Mark them so the flat loop below does - // not also emit them as stray top-level paragraphs. + // not also Emit them as stray top-level paragraphs. consumed := tableDescendants(blocks, byID) var sb strings.Builder @@ -40,49 +40,49 @@ func blocksToMarkdown(ctx context.Context, client sheetReader, blocks []docxBloc continue } switch b.BlockType { - case blockTypePage: + case BlockTypePage: continue // root container, no text of its own - case blockTypeText: + case BlockTypeText: writePara(&sb, plainText(textBearingField(b))) - case blockTypeBullet: + case BlockTypeBullet: writePara(&sb, "- "+plainText(textBearingField(b))) - case blockTypeOrdered: + case BlockTypeOrdered: writePara(&sb, "1. "+plainText(textBearingField(b))) - case blockTypeCode: + case BlockTypeCode: writePara(&sb, "```\n"+plainText(textBearingField(b))+"\n```") - case blockTypeQuote: + case BlockTypeQuote: writePara(&sb, "> "+plainText(textBearingField(b))) - case blockTypeDivider: + case BlockTypeDivider: writePara(&sb, "---") - case blockTypeTable: + case BlockTypeTable: writePara(&sb, renderNativeTable(b, byID)) - case blockTypeTableCell: + case BlockTypeTableCell: continue // rendered by its parent table (also covered by `consumed`) - case blockTypeSheet: + case BlockTypeSheet: if b.Sheet != nil { writePara(&sb, inlineTable(ctx, client, b.Sheet.Token, "sheet")) } - case blockTypeBitable: + case BlockTypeBitable: if b.Bitable != nil { writePara(&sb, inlineTable(ctx, client, b.Bitable.Token, "bitable")) } - case blockTypeImage: + case BlockTypeImage: // Emit a token-free placeholder: images carry no retrievable text, and // leaking the internal media token would pollute embeddings. A neutral // marker preserves surrounding context (e.g. "如下图所示"). writePara(&sb, "![图片]()") - case blockTypeTodo: + case BlockTypeTodo: if t := plainText(textBearingField(b)); t != "" { writePara(&sb, "- [ ] "+t) } - case blockTypeCallout: + case BlockTypeCallout: // Callout is a container; its body is usually in child blocks (rendered // separately). Emit its own text only if it carries direct inline text, // so the container case is a safe no-op. if t := plainText(textBearingField(b)); t != "" { writePara(&sb, "> "+t) } - case blockTypeFile: + case BlockTypeFile: if b.File != nil { name := b.File.Name if name == "" { @@ -92,8 +92,8 @@ func blocksToMarkdown(ctx context.Context, client sheetReader, blocks []docxBloc atts = append(atts, pendingAttachment{FileToken: b.File.Token, Name: b.File.Name}) } default: - if b.BlockType >= blockTypeHeading1 && b.BlockType <= blockTypeHeading9 { - level := b.BlockType - blockTypeHeading1 + 1 + if b.BlockType >= BlockTypeHeading1 && b.BlockType <= blockTypeHeading9 { + level := b.BlockType - BlockTypeHeading1 + 1 writePara(&sb, strings.Repeat("#", level)+" "+plainText(textBearingField(b))) } } @@ -116,7 +116,7 @@ func writePara(sb *strings.Builder, s string) { } // plainText concatenates the text runs of a text-bearing block. -func plainText(bt *blockText) string { +func plainText(bt *BlockText) string { if bt == nil { return "" } @@ -130,12 +130,12 @@ func plainText(bt *blockText) string { } // headingText returns the heading field for the block's level, or Text as fallback. -func headingText(b docxBlock) *blockText { - fields := []*blockText{ +func headingText(b DocxBlock) *BlockText { + fields := []*BlockText{ b.Heading1, b.Heading2, b.Heading3, b.Heading4, b.Heading5, b.Heading6, b.Heading7, b.Heading8, b.Heading9, } - if idx := b.BlockType - blockTypeHeading1; idx >= 0 && idx < len(fields) && fields[idx] != nil { + if idx := b.BlockType - BlockTypeHeading1; idx >= 0 && idx < len(fields) && fields[idx] != nil { return fields[idx] } return b.Text @@ -145,7 +145,7 @@ func headingText(b docxBlock) *blockText { // every cell listed in a table block plus everything reachable through those // cells' Children. blocksToMarkdown skips these so table content is emitted only // by the table renderer, never a second time as loose paragraphs. -func tableDescendants(blocks []docxBlock, byID map[string]docxBlock) map[string]bool { +func tableDescendants(blocks []DocxBlock, byID map[string]DocxBlock) map[string]bool { consumed := make(map[string]bool) var mark func(id string) mark = func(id string) { @@ -156,7 +156,7 @@ func tableDescendants(blocks []docxBlock, byID map[string]docxBlock) map[string] // Attachment/media blocks nested in a cell must still be collected (and // their reference emitted) by the main loop — the table renderer only // extracts text — so do not consume them, only their text structure. - if b.BlockType == blockTypeFile || b.BlockType == blockTypeImage { + if b.BlockType == BlockTypeFile || b.BlockType == BlockTypeImage { return } consumed[id] = true @@ -168,8 +168,8 @@ func tableDescendants(blocks []docxBlock, byID map[string]docxBlock) map[string] // Only consume cells of tables we will actually render. A table that // renderNativeTable would bail on (missing property / zero columns) must // NOT have its cells consumed, or their text would be dropped entirely — - // leave them for the flat loop to emit as loose paragraphs instead. - if b.BlockType == blockTypeTable && tableRenderable(b) { + // leave them for the flat loop to Emit as loose paragraphs instead. + if b.BlockType == BlockTypeTable && tableRenderable(b) { for _, cid := range b.Table.Cells { mark(cid) } @@ -182,31 +182,31 @@ func tableDescendants(blocks []docxBlock, byID map[string]docxBlock) map[string] // (a column count) for renderNativeTable to produce a Markdown table. It is the // single predicate shared by the consume pass and the render pass so the two // never disagree about which tables are handled by the table renderer. -func tableRenderable(b docxBlock) bool { +func tableRenderable(b DocxBlock) bool { return b.Table != nil && b.Table.Property != nil && b.Table.Property.ColumnSize > 0 } // textBearingField returns the inline-text payload for whichever type-named // field a block populates (docx stores a block's text in a field named after // its type), so cell content of any text-like type can be extracted uniformly. -func textBearingField(b docxBlock) *blockText { +func textBearingField(b DocxBlock) *BlockText { switch b.BlockType { - case blockTypeText: + case BlockTypeText: return b.Text - case blockTypeBullet: + case BlockTypeBullet: return b.Bullet - case blockTypeOrdered: + case BlockTypeOrdered: return b.Ordered - case blockTypeCode: + case BlockTypeCode: return b.Code - case blockTypeQuote: + case BlockTypeQuote: return b.Quote - case blockTypeTodo: + case BlockTypeTodo: return b.Todo - case blockTypeCallout: + case BlockTypeCallout: return b.Callout } - if b.BlockType >= blockTypeHeading1 && b.BlockType <= blockTypeHeading9 { + if b.BlockType >= BlockTypeHeading1 && b.BlockType <= blockTypeHeading9 { return headingText(b) } return b.Text @@ -215,7 +215,7 @@ func textBearingField(b docxBlock) *blockText { // cellText renders a native table cell to a single string. A Feishu table_cell // (block_type 32) is a container: its text lives in child blocks, not on the // cell itself, so we concatenate the text of each child block. -func cellText(cell docxBlock, byID map[string]docxBlock) string { +func cellText(cell DocxBlock, byID map[string]DocxBlock) string { var parts []string for _, childID := range cell.Children { if t := plainText(textBearingField(byID[childID])); t != "" { @@ -226,7 +226,7 @@ func cellText(cell docxBlock, byID map[string]docxBlock) string { } // renderNativeTable renders a native docx table block into a Markdown table. -func renderNativeTable(b docxBlock, byID map[string]docxBlock) string { +func renderNativeTable(b DocxBlock, byID map[string]DocxBlock) string { if !tableRenderable(b) { return "" } @@ -248,7 +248,7 @@ func renderNativeTable(b docxBlock, byID map[string]docxBlock) string { // markdownTable renders a [][]string (first row = header) as a GFM table. func markdownTable(rows [][]string) string { - // A zero-column header (an embedded sheet/bitable with no columns) would emit + // A zero-column header (an embedded sheet/bitable with no columns) would Emit // a header line of "| |" and a separator of just "|" — malformed GFM. Render // nothing instead. if len(rows) == 0 || len(rows[0]) == 0 { @@ -259,7 +259,7 @@ func markdownTable(rows [][]string) string { sb.WriteString("| " + strings.Join(escapePipes(rows[0]), " | ") + " |\n") sb.WriteString("|" + strings.Repeat(" --- |", cols) + "\n") for _, r := range rows[1:] { - // Ragged data: a row wider than the header would emit more cells than the + // Ragged data: a row wider than the header would Emit more cells than the // header/separator declare, producing a malformed GFM table. Clamp to the // header width (truncate overflow, pad shortfall). if len(r) > cols { @@ -297,10 +297,10 @@ func inlineTable(ctx context.Context, client sheetReader, token, kind string) st noun string ) if kind == "sheet" { - rows, truncated, err = client.ReadSheetRange(ctx, token) + rows, truncated, err = client.readSheetRange(ctx, token) noun = "内嵌电子表格" } else { - rows, truncated, err = client.ReadBitableRecords(ctx, token) + rows, truncated, err = client.readBitableRecords(ctx, token) noun = "内嵌多维表格" } if err != nil { diff --git a/internal/datasource/connector/feishu/markdown_test.go b/internal/datasource/connector/feishu/core/markdown_test.go similarity index 72% rename from internal/datasource/connector/feishu/markdown_test.go rename to internal/datasource/connector/feishu/core/markdown_test.go index ed29d4068..579ec8020 100644 --- a/internal/datasource/connector/feishu/markdown_test.go +++ b/internal/datasource/connector/feishu/core/markdown_test.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "context" @@ -7,11 +7,9 @@ import ( "testing" ) -// txt builds a text-bearing blockText from plain content. -func txt(s string) *blockText { - return &blockText{Elements: []textElement{{TextRun: &struct { - Content string `json:"content"` - }{Content: s}}}} +// txt builds a text-bearing BlockText from plain content. +func txt(s string) *BlockText { + return &BlockText{Elements: []TextElement{{TextRun: &TextRun{Content: s}}}} } type fakeReader struct { @@ -23,26 +21,20 @@ type fakeReader struct { bitableErr error } -func (f fakeReader) ReadSheetRange(_ context.Context, _ string) ([][]string, bool, error) { +func (f fakeReader) readSheetRange(_ context.Context, _ string) ([][]string, bool, error) { return f.sheet, f.sheetTruncated, f.sheetErr } -func (f fakeReader) ReadBitableRecords(_ context.Context, _ string) ([][]string, bool, error) { + +func (f fakeReader) readBitableRecords(_ context.Context, _ string) ([][]string, bool, error) { return f.bitable, f.bitableTruncated, f.bitableErr } func TestBlocksToMarkdown_EmbeddedSheetAndFile(t *testing.T) { - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, - {BlockID: "s", BlockType: blockTypeSheet, Sheet: &struct { - Token string `json:"token"` - }{Token: "sht_a_0"}}, - {BlockID: "img", BlockType: blockTypeImage, Image: &struct { - Token string `json:"token"` - }{Token: "img_t"}}, - {BlockID: "f", BlockType: blockTypeFile, File: &struct { - Token string `json:"token"` - Name string `json:"name"` - }{Token: "file_t", Name: "报表.pdf"}}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, + {BlockID: "s", BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: "sht_a_0"}}, + {BlockID: "img", BlockType: BlockTypeImage, Image: &BlockTokenRef{Token: "img_t"}}, + {BlockID: "f", BlockType: BlockTypeFile, File: &BlockFileRef{Token: "file_t", Name: "报表.pdf"}}, } fr := fakeReader{sheet: [][]string{{"名称", "数量"}, {"苹果", "3"}}} md, atts, err := blocksToMarkdown(context.Background(), fr, blocks) @@ -67,11 +59,9 @@ func TestBlocksToMarkdown_EmbeddedSheetAndFile(t *testing.T) { } func TestBlocksToMarkdown_SheetTruncatedNote(t *testing.T) { - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, - {BlockID: "s", BlockType: blockTypeSheet, Sheet: &struct { - Token string `json:"token"` - }{Token: "sht_a_0"}}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, + {BlockID: "s", BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: "sht_a_0"}}, } fr := fakeReader{sheet: [][]string{{"h"}, {"1"}}, sheetTruncated: true} md, _, err := blocksToMarkdown(context.Background(), fr, blocks) @@ -84,11 +74,9 @@ func TestBlocksToMarkdown_SheetTruncatedNote(t *testing.T) { } func TestBlocksToMarkdown_SheetPermissionDegrades(t *testing.T) { - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, - {BlockID: "s", BlockType: blockTypeSheet, Sheet: &struct { - Token string `json:"token"` - }{Token: "sht_a_0"}}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, + {BlockID: "s", BlockType: BlockTypeSheet, Sheet: &BlockTokenRef{Token: "sht_a_0"}}, } fr := fakeReader{sheetErr: fmt.Errorf("code=99991672 permission denied")} md, _, err := blocksToMarkdown(context.Background(), fr, blocks) @@ -102,11 +90,9 @@ func TestBlocksToMarkdown_SheetPermissionDegrades(t *testing.T) { func TestBlocksToMarkdown_BitableInlinedAndDegrades(t *testing.T) { mk := func(fr fakeReader) string { - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, - {BlockID: "bt", BlockType: blockTypeBitable, Bitable: &struct { - Token string `json:"token"` - }{Token: "bascabc_tblxyz"}}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, + {BlockID: "bt", BlockType: BlockTypeBitable, Bitable: &BlockTokenRef{Token: "bascabc_tblxyz"}}, } md, _, err := blocksToMarkdown(context.Background(), fr, blocks) if err != nil { @@ -129,9 +115,9 @@ func TestBlocksToMarkdown_BitableInlinedAndDegrades(t *testing.T) { func TestBlocksToMarkdown_NativeTableFromCellChildren(t *testing.T) { // Real Feishu shape: a table_cell (block_type 32) is a container whose text // lives in child text blocks, not on the cell. The renderer must read cell - // text from those children AND must not also emit them as loose paragraphs. - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, + // text from those children AND must not also Emit them as loose paragraphs. + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, tableBlk("t", 2, "c1", "c2", "c3", "c4"), cellBlk("c1"), cellBlk("c2"), cellBlk("c3"), cellBlk("c4"), cellTextBlk("c1", "姓名"), cellTextBlk("c2", "分数"), @@ -155,14 +141,11 @@ func TestBlocksToMarkdown_AttachmentInsideTableCellStillCollected(t *testing.T) // A file block nested inside a table cell must still be collected as an // attachment — the table "consumed" set marks a cell's text structure but // must NOT swallow attachment/media blocks, or embedded files silently vanish. - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, tableBlk("t", 1, "c1"), - {BlockID: "c1", BlockType: blockTypeTableCell, Children: []string{"f1"}}, - {BlockID: "f1", BlockType: blockTypeFile, File: &struct { - Token string `json:"token"` - Name string `json:"name"` - }{Token: "tok-in-cell", Name: "内嵌.pdf"}}, + {BlockID: "c1", BlockType: BlockTypeTableCell, Children: []string{"f1"}}, + {BlockID: "f1", BlockType: BlockTypeFile, File: &BlockFileRef{Token: "tok-in-cell", Name: "内嵌.pdf"}}, } _, atts, err := blocksToMarkdown(context.Background(), nil, blocks) if err != nil { @@ -201,7 +184,7 @@ func TestMarkdownTable_RaggedRowClampedToHeader(t *testing.T) { func TestMarkdownTable_ZeroColumnRendersNothing(t *testing.T) { // An embedded sheet/bitable with a header row but no columns (e.g. a bitable // whose fields were all deleted) yields rows == [][]string{{}}. len(rows)==1 - // passes a naive empty guard, but cols==0 would emit a malformed GFM table + // passes a naive empty guard, but cols==0 would Emit a malformed GFM table // ("| |" header + a bare "|" separator). It must render nothing instead. for _, rows := range [][][]string{ {{}}, // one empty header row, no data @@ -219,16 +202,11 @@ func TestBlocksToMarkdown_UnrenderableTablePreservesCellText(t *testing.T) { // cannot be rendered as a Markdown table. Its cells must NOT be consumed, so // their text still reaches the output as loose paragraphs rather than // vanishing entirely. - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, // tableBlk sets Property (renderable); here we want an UNrenderable one. - {BlockID: "t", BlockType: blockTypeTable, Table: &struct { - Cells []string `json:"cells"` - Property *struct { - ColumnSize int `json:"column_size"` - } `json:"property"` - }{Cells: []string{"c1"}}}, - {BlockID: "c1", BlockType: blockTypeTableCell, Children: []string{"c1_txt"}}, + {BlockID: "t", BlockType: BlockTypeTable, Table: &BlockTable{Cells: []string{"c1"}}}, + {BlockID: "c1", BlockType: BlockTypeTableCell, Children: []string{"c1_txt"}}, cellTextBlk("c1", "重要内容"), } md, _, err := blocksToMarkdown(context.Background(), nil, blocks) @@ -241,11 +219,11 @@ func TestBlocksToMarkdown_UnrenderableTablePreservesCellText(t *testing.T) { } func TestBlocksToMarkdown_TextConstructs(t *testing.T) { - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage, Children: []string{"h", "p", "b"}}, - {BlockID: "h", BlockType: blockTypeHeading1, Heading1: txt("标题")}, - {BlockID: "p", BlockType: blockTypeText, Text: txt("一段正文")}, - {BlockID: "b", BlockType: blockTypeBullet, Bullet: txt("要点")}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage, Children: []string{"h", "p", "b"}}, + {BlockID: "h", BlockType: BlockTypeHeading1, Heading1: txt("标题")}, + {BlockID: "p", BlockType: BlockTypeText, Text: txt("一段正文")}, + {BlockID: "b", BlockType: BlockTypeBullet, Bullet: txt("要点")}, } md, atts, err := blocksToMarkdown(context.Background(), nil, blocks) if err != nil { @@ -262,14 +240,14 @@ func TestBlocksToMarkdown_TextConstructs(t *testing.T) { func TestBlocksToMarkdown_BlankDocRendersEmpty(t *testing.T) { // A page with no renderable children (or only empty-text/container blocks) - // must render to empty Markdown. fetchDocxWithBlocks relies on this: an empty + // must render to empty Markdown. FetchDocxWithBlocks relies on this: an empty // render triggers the export fallback instead of emitting a content-less main // item that would wrongly ingest the login-gated wiki URL. Any File/Image // block writes a placeholder, so a truly empty render also implies no // attachment/image downdrill is lost by that fallback. - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage, Children: []string{"p"}}, - {BlockID: "p", BlockType: blockTypeText, Text: txt("")}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage, Children: []string{"p"}}, + {BlockID: "p", BlockType: BlockTypeText, Text: txt("")}, } md, atts, err := blocksToMarkdown(context.Background(), nil, blocks) if err != nil { @@ -284,10 +262,10 @@ func TestBlocksToMarkdown_BlankDocRendersEmpty(t *testing.T) { } func TestBlocksToMarkdown_TodoAndCallout(t *testing.T) { - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, - {BlockID: "t", BlockType: blockTypeTodo, Todo: txt("买牛奶")}, - {BlockID: "c", BlockType: blockTypeCallout, Callout: txt("注意事项")}, + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, + {BlockID: "t", BlockType: BlockTypeTodo, Todo: txt("买牛奶")}, + {BlockID: "c", BlockType: BlockTypeCallout, Callout: txt("注意事项")}, } md, _, err := blocksToMarkdown(context.Background(), nil, blocks) if err != nil { @@ -302,16 +280,16 @@ func TestBlocksToMarkdown_TodoAndCallout(t *testing.T) { } func TestBlocksToMarkdown_CalloutContainerNoOp(t *testing.T) { - // A callout with no direct text (container form) must emit nothing itself. - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, - {BlockID: "c", BlockType: blockTypeCallout}, + // A callout with no direct text (container form) must Emit nothing itself. + blocks := []DocxBlock{ + {BlockID: "root", BlockType: BlockTypePage}, + {BlockID: "c", BlockType: BlockTypeCallout}, } md, _, err := blocksToMarkdown(context.Background(), nil, blocks) if err != nil { t.Fatalf("err: %v", err) } if strings.TrimSpace(string(md)) != "" { - t.Errorf("empty callout should emit nothing, got:\n%q", md) + t.Errorf("empty callout should Emit nothing, got:\n%q", md) } } diff --git a/internal/datasource/connector/feishu/region.go b/internal/datasource/connector/feishu/core/region.go similarity index 64% rename from internal/datasource/connector/feishu/region.go rename to internal/datasource/connector/feishu/core/region.go index b2898dcf0..551355fd8 100644 --- a/internal/datasource/connector/feishu/region.go +++ b/internal/datasource/connector/feishu/core/region.go @@ -1,4 +1,4 @@ -package feishu +package core import "github.com/Tencent/WeKnora/internal/types" @@ -48,10 +48,34 @@ var ( WebBaseURL: larkWebBaseURL, Label: "Lark", } + + // RegionFeishuDrive is the Chinese mainland cloud, Drive (云盘) mode. + // Shares the feishu connector package with RegionFeishu; only the connector + // type differs so the registry dispatches to the Drive connector. + RegionFeishuDrive = Region{ + ConnectorType: types.ConnectorTypeFeishuDrive, + OpenBaseURL: feishuOpenBaseURL, + WebBaseURL: feishuWebBaseURL, + Label: "FeishuDrive", + } + + // RegionLarkDrive is the international cloud, Drive mode. + RegionLarkDrive = Region{ + ConnectorType: types.ConnectorTypeLarkDrive, + OpenBaseURL: larkOpenBaseURL, + WebBaseURL: larkWebBaseURL, + Label: "LarkDrive", + } ) -// wikiURL builds the user-facing link to a wiki space or node on this cloud. +// WikiURL builds the user-facing link to a wiki space or node on this cloud. // The token is either a space_id or a node_token; both live under /wiki/. -func (r Region) wikiURL(token string) string { +func (r Region) WikiURL(token string) string { return r.WebBaseURL + "/wiki/" + token } + +// DriveFolderURL builds the user-facing link to a Drive folder on this cloud. +// A folder_token lives under /drive/folder/. +func (r Region) DriveFolderURL(folderToken string) string { + return r.WebBaseURL + "/drive/folder/" + folderToken +} diff --git a/internal/datasource/connector/feishu/region_test.go b/internal/datasource/connector/feishu/core/region_test.go similarity index 72% rename from internal/datasource/connector/feishu/region_test.go rename to internal/datasource/connector/feishu/core/region_test.go index f6cbab899..517a46401 100644 --- a/internal/datasource/connector/feishu/region_test.go +++ b/internal/datasource/connector/feishu/core/region_test.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "strings" @@ -25,27 +25,21 @@ func TestRegions_AreDistinct(t *testing.T) { } } -func TestConnectorType_FollowsRegion(t *testing.T) { - if got := NewConnector(RegionFeishu).Type(); got != types.ConnectorTypeFeishu { - t.Errorf("Feishu connector Type() = %q, want %q", got, types.ConnectorTypeFeishu) - } - if got := NewConnector(RegionLark).Type(); got != types.ConnectorTypeLark { - t.Errorf("Lark connector Type() = %q, want %q", got, types.ConnectorTypeLark) - } -} +// TestConnectorType_FollowsRegion lives in package wiki (it exercises +// wiki.NewConnector); see wiki/connector_test.go TestConnectorType. // The wiki link shown in the resource picker must point at the cloud the data // actually lives on. Feishu's value is the pre-existing one and must not drift. func TestRegion_WikiURL(t *testing.T) { - if got, want := RegionFeishu.wikiURL("spc123"), "https://feishu.cn/wiki/spc123"; got != want { - t.Errorf("Feishu wikiURL = %q, want %q", got, want) + if got, want := RegionFeishu.WikiURL("spc123"), "https://feishu.cn/wiki/spc123"; got != want { + t.Errorf("Feishu WikiURL = %q, want %q", got, want) } - if got, want := RegionLark.wikiURL("spc123"), "https://larksuite.com/wiki/spc123"; got != want { - t.Errorf("Lark wikiURL = %q, want %q", got, want) + if got, want := RegionLark.WikiURL("spc123"), "https://larksuite.com/wiki/spc123"; got != want { + t.Errorf("Lark WikiURL = %q, want %q", got, want) } // A Lark link must never point at the Feishu host — that was the bug. - if strings.Contains(RegionLark.wikiURL("x"), "feishu") { - t.Errorf("Lark wikiURL leaks the Feishu host: %q", RegionLark.wikiURL("x")) + if strings.Contains(RegionLark.WikiURL("x"), "feishu") { + t.Errorf("Lark WikiURL leaks the Feishu host: %q", RegionLark.WikiURL("x")) } } @@ -70,9 +64,9 @@ func TestParseFeishuConfig_BaseURLDefaultsToRegion(t *testing.T) { if c.baseURL != "" { creds["base_url"] = c.baseURL } - cfg, err := parseFeishuConfig(&types.DataSourceConfig{Credentials: creds}, c.region) + cfg, err := ParseFeishuConfig(&types.DataSourceConfig{Credentials: creds}, c.region) if err != nil { - t.Fatalf("parseFeishuConfig: %v", err) + t.Fatalf("ParseFeishuConfig: %v", err) } if got := cfg.GetBaseURL(); got != c.want { t.Errorf("GetBaseURL() = %q, want %q", got, c.want) diff --git a/internal/datasource/connector/feishu/core/shared.go b/internal/datasource/connector/feishu/core/shared.go new file mode 100644 index 000000000..5900ecf09 --- /dev/null +++ b/internal/datasource/connector/feishu/core/shared.go @@ -0,0 +1,502 @@ +package core + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "net/http" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +const FeishuWikiNodeResourceSeparator = ":" + +// shared.go holds the helpers used by BOTH the wiki Connector (connector.go) +// and the Drive DriveConnector (connector.go): error classification, +// config parsing, stream-Checkpoint tuning, the fetch tally, filename/time +// utilities, attachment rules, and the docx blocks fetch path. Anything that +// is specific to one connector stays in that connector's own file. + +// FeishuStreamCheckpointInterval is how many processed nodes pass between +// cursor checkpoints during a streaming fetch. Small enough that a timed-out +// sync loses little work on resume, large enough that Checkpoint persistence +// (a DB write) does not dominate. Overridable in tests. See FetchStream. +var FeishuStreamCheckpointInterval = 50 + +// FeishuStreamCheckpointMaxInterval bounds checkpointing by wall-clock time as +// well as node count. Without it, a sync of fewer than +// FeishuStreamCheckpointInterval very slow (rate-limited) exports could reach +// the 2h task timeout having never checkpointed, and resume from scratch every +// retry — the #2136 "never fully syncs" case. Overridable in tests. +var FeishuStreamCheckpointMaxInterval = 30 * time.Second + +// fetchTally accumulates the outcome of fetching a wiki node subtree so the +// connector can Emit a single actionable summary. Without it, unsupported nodes +// (mindnote/slides/etc.) vanish with no item, no error and no log, leaving users +// unable to explain why "13 documents synced only 3" (Tencent/WeKnora#2136). +type fetchTally struct { + discovered int + fetched int + failed int + skippedByType map[string]int +} + +func newFetchTally(discovered int) *fetchTally { + return &fetchTally{discovered: discovered, skippedByType: map[string]int{}} +} + +func (t *fetchTally) fetch() { t.fetched++ } +func (t *fetchTally) fail() { t.failed++ } +func (t *fetchTally) Skip(objType string) { t.skippedByType[objType]++ } + +func (t *fetchTally) skipped() int { + n := 0 + for _, c := range t.skippedByType { + n += c + } + return n +} + +func (t *fetchTally) summary() string { + return fmt.Sprintf("discovered=%d fetched=%d failed=%d skipped_unsupported=%d by_type=%v", + t.discovered, t.fetched, t.failed, t.skipped(), t.skippedByType) +} + +var reFeishuErrorCode = regexp.MustCompile(`code["\s]*[:=]\s*(\d+)`) + +// feishuErrorCode extracts the numeric Feishu error code from a raw error string +// (e.g. `body={"code":1663,...}` or `code=1663`), best-effort. +func feishuErrorCode(raw string) string { + if m := reFeishuErrorCode.FindStringSubmatch(raw); len(m) == 2 { + return m[1] + } + return "" +} + +// feishuFailure classifies a raw connector/API error into a stable i18n code +// (mapped to a localized string on the frontend), an optional numeric Feishu +// error code for interpolation, and an English fallback message for clients +// without the i18n key. The raw status/JSON body/log_id is never returned here — +// it stays in the server logs. Dumping it in the UI is the anti-pattern +// Airbyte/Fivetran/Onyx warn against. Transient errors are retried next sync +// (the cursor is retained); auth/permission errors point at the fix instead. +func feishuFailure(err error) (code, codeValue, fallback string) { + if err == nil { + return "sync_failed", "", "Sync failed; will retry on the next sync" + } + s := strings.ToLower(err.Error()) + + switch { + case strings.Contains(s, "auth error"), + strings.Contains(s, "invalid access token"), + strings.Contains(s, "permission"), + strings.Contains(s, "forbidden"), + strings.Contains(s, "status=403"): + return "feishu_auth_or_permission", "", "Authentication or permission error; check credentials and app scopes" + case strings.Contains(s, "rate limited"), strings.Contains(s, "status=429"): + return "feishu_rate_limited", "", "Feishu API rate limited; will retry on the next sync" + case strings.Contains(s, "timed out"), + strings.Contains(s, "timeout"), + strings.Contains(s, "deadline exceeded"): + return "feishu_timeout", "", "Export or request timed out; will retry on the next sync" + case strings.Contains(s, "server error"): + return "feishu_server_unavailable", "", "Feishu service temporarily unavailable; will retry on the next sync" + case strings.Contains(s, "api error"), + strings.Contains(s, "export task failed"), + strings.Contains(s, "download failed"): + if v := feishuErrorCode(err.Error()); v != "" { + return "feishu_api_error", v, fmt.Sprintf("Feishu API error (code=%s); will retry on the next sync", v) + } + return "feishu_api_error_generic", "", "Feishu API error; will retry on the next sync" + default: + return "sync_failed", "", "Sync failed; will retry on the next sync" + } +} + +// FeishuErrorItemMeta builds the metadata for a failed item: the raw error (for +// server logs) plus the classified i18n code / param / fallback (for a +// localisable SyncItemError in the UI), merged with any caller-supplied extras. +func FeishuErrorItemMeta(err error, extra map[string]string) map[string]string { + code, codeValue, fallback := feishuFailure(err) + m := map[string]string{ + "error": err.Error(), + "error_reason_code": code, + "error_reason": fallback, + } + if codeValue != "" { + m["error_reason_code_value"] = codeValue + } + maps.Copy(m, extra) + return m +} + +// parseableAttachmentExts are attachment extensions worth ingesting as their +// own knowledge entries; other files (icons, tiny decor) are skipped. +var parseableAttachmentExts = map[string]bool{ + ".pdf": true, ".doc": true, ".docx": true, ".xls": true, ".xlsx": true, + ".ppt": true, ".pptx": true, ".txt": true, ".md": true, ".csv": true, +} + +// MinAttachmentBytes filters out decorative micro-files. +const MinAttachmentBytes = 2 * 1024 + +// SupportedImageExt sniffs image bytes and returns the filename extension and +// content type WeKnora accepts for a standalone image knowledge item (png/jpg/ +// gif — the image set isValidFileType admits). ok is false for non-image or +// unsupported formats (e.g. webp/bmp), which the caller skips rather than +// mislabel — a wrong extension would fail parsing. The detected content type is +// returned even when ok is false so the caller can log it without re-sniffing. +func SupportedImageExt(data []byte) (ext, contentType string, ok bool) { + switch ct := http.DetectContentType(data); ct { + case "image/png": + return ".png", ct, true + case "image/jpeg": + return ".jpg", ct, true + case "image/gif": + return ".gif", ct, true + default: + return "", ct, false + } +} + +// ParseFeishuConfig extracts and validates Feishu/Lark-specific configuration. +// +// base_url stays an explicit override so existing data sources that pointed a +// "feishu" connector at open.larksuite.com keep working; when it is unset the +// region's own host is filled in, making the resolved Config.BaseURL concrete +// for everything downstream. +func ParseFeishuConfig(config *types.DataSourceConfig, region Region) (*Config, error) { + if config == nil { + return nil, fmt.Errorf("config is nil") + } + + credBytes, err := json.Marshal(config.Credentials) + if err != nil { + return nil, fmt.Errorf("marshal credentials: %w", err) + } + + var feishuConfig Config + if err := json.Unmarshal(credBytes, &feishuConfig); err != nil { + return nil, fmt.Errorf("parse %s credentials: %w", region.ConnectorType, err) + } + + if feishuConfig.AppID == "" || feishuConfig.AppSecret == "" { + return nil, fmt.Errorf("%s app_id and app_secret are required", region.ConnectorType) + } + + if feishuConfig.BaseURL == "" { + feishuConfig.BaseURL = region.OpenBaseURL + } + + // Timezone is a display setting (bitable date rendering), not a credential, so + // it lives in Settings. Empty falls back to GMT+8 in resolveLocation. + if feishuConfig.Timezone == "" && config.Settings != nil { + if tz, ok := config.Settings["timezone"].(string); ok { + feishuConfig.Timezone = strings.TrimSpace(tz) + } + } + + if err := datasource.ValidateConnectorBaseURL(feishuConfig.GetBaseURL()); err != nil { + return nil, err + } + + return &feishuConfig, nil +} + +// IsSupportedDocType checks if a Feishu document type can be synced. +// mindnote and slides have no content read API and are skipped. +func IsSupportedDocType(objType string) bool { + switch objType { + case "docx", "doc", "sheet", "bitable", "file": + return true + default: + // mindnote, slides — no content retrieval API available + return false + } +} + +// ParseFeishuTimestamp parses a Feishu unix timestamp string (seconds) into time.Time. +func ParseFeishuTimestamp(ts string) time.Time { + if ts == "" { + return time.Time{} + } + sec, err := strconv.ParseInt(ts, 10, 64) + if err != nil { + return time.Time{} + } + return time.Unix(sec, 0) +} + +// SanitizeFileName removes characters that are invalid in filenames and +// truncates at a UTF-8 rune boundary. Raw byte truncation would split a +// multi-byte codepoint (Chinese chars are 3 bytes) and produce invalid UTF-8 +// that downstream validation (utf8.ValidString) rejects. +// +// The extension is preserved across truncation: only the base name is trimmed, +// so a long attachment name like "很长的名字….pdf" keeps its ".pdf" suffix that +// downstream file-type classification depends on. +func SanitizeFileName(name string) string { + if name == "" { + return "untitled" + } + replacer := strings.NewReplacer( + "/", "_", "\\", "_", ":", "_", "*", "_", + "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", + ) + result := replacer.Replace(name) + const maxBytes = 200 + if len(result) <= maxBytes { + return result + } + ext := filepath.Ext(result) + if len(ext) >= maxBytes { + // pathological: extension alone overflows the budget → drop it + ext = "" + } + base := truncateUTF8(result[:len(result)-len(ext)], maxBytes-len(ext)) + return base + ext +} + +// truncateUTF8 shortens s to at most maxBytes bytes without splitting a +// multi-byte rune: after a hard byte cut it trims any trailing partial codepoint. +func truncateUTF8(s string, maxBytes int) string { + if len(s) <= maxBytes { + return s + } + s = s[:maxBytes] + for len(s) > 0 { + r, size := utf8.DecodeLastRuneInString(s) + if r != utf8.RuneError || size != 1 { + break + } + s = s[:len(s)-1] + } + return s +} + +// DocxFetchInput is the unified description of one docx document from either +// source (wiki node or Drive file) that FetchDocxWithBlocks needs. +type DocxFetchInput struct { + // WeKnora external_id: wiki=node.NodeToken, drive=file.Token + DocToken string + // Feishu docx document token + ObjToken string + Title string + URL string + ResourceID string + EditTime time.Time + BaseMeta map[string]string + MultimodalEnabled bool +} + +// FetchDocxWithBlocks retrieves a docx document via the blocks API, converts it +// to Markdown, and returns a main item plus any parseable attachment/image +// sub-items. Falls back to the export API if the blocks API errors or renders +// empty. Shared by the wiki Connector and the Drive DriveConnector. +func FetchDocxWithBlocks(ctx context.Context, client *Client, in DocxFetchInput) ([]*types.FetchedItem, error) { + // FEISHU_DOCX_PARSE_MODE selects the docx parsing path. The blocks path + // renders image blocks as empty `![图片]()` placeholders and fans images out + // into separate knowledge items, which breaks image↔document association in + // retrieval/wiki/agent. The export path yields a .docx that docreader parses + // inline, so images are bound to the parent document via parent_chunk_id + // (same as a regular docx upload). Default (unset / "export") uses export so + // images associate with the document; set "blocks" for the blocks-first + // behaviour (faster, keeps docx attachments, but images are detached). + + // This is a temporary solution. If a better parsing solution is available later, this environment variable will be removed and replaced with a better one. + parsingMode := strings.TrimSpace(os.Getenv("FEISHU_DOCX_PARSE_MODE")) + if parsingMode == "" { + parsingMode = "export" + } + + if strings.EqualFold(parsingMode, "export") { + item, err := exportDocxFallback(ctx, client, in) + if err != nil { + return nil, err + } + return []*types.FetchedItem{item}, nil + } + + blocks, err := client.listDocumentBlocks(ctx, in.ObjToken) + if err != nil { + logger.Warnf(ctx, "[Feishu] blocks API failed for %s (%s), falling back to export: %v", + in.Title, in.ObjToken, err) + item, ferr := exportDocxFallback(ctx, client, in) + if ferr != nil { + return nil, ferr + } + // Do NOT set ReplacesSubtree here (see the wiki history: a transient + // blocks failure must not sweep good attachment children from the prior + // blocks-path sync with nothing to replace them). + return []*types.FetchedItem{item}, nil + } + + md, atts, err := blocksToMarkdown(ctx, client, blocks) + if err != nil { + return nil, fmt.Errorf("convert blocks %s: %w", in.Title, err) + } + + if len(strings.TrimSpace(string(md))) == 0 { + logger.Infof(ctx, "[Feishu] doc %s (%s): blocks rendered empty Markdown, falling back to export", + in.Title, in.ObjToken) + item, ferr := exportDocxFallback(ctx, client, in) + if ferr != nil { + return nil, ferr + } + return []*types.FetchedItem{item}, nil + } + + main := &types.FetchedItem{ + ExternalID: in.DocToken, + Title: in.Title, + Content: md, + ContentType: "text/markdown", + FileName: SanitizeFileName(in.Title) + ".md", + URL: in.URL, + UpdatedAt: in.EditTime, + SourceResourceID: in.ResourceID, + Metadata: in.BaseMeta, + ReplacesSubtree: true, // sweep stale attachment sub-items on re-sync + } + items := []*types.FetchedItem{main} + + keep := make([]string, 0, len(atts)) + childMeta := func() map[string]string { + m := maps.Clone(in.BaseMeta) + m["parent_node_token"] = in.DocToken + m["attachment"] = "true" + return m + } + for _, a := range atts { + childID := types.SubtreeChildID(in.DocToken, "file", a.FileToken) + keep = append(keep, childID) // present in the doc → never sweep as stale + ext := strings.ToLower(filepath.Ext(a.Name)) + if ext == "" { + logger.Warnf(ctx, "[Feishu] doc %s: skipping attachment with no usable filename (token=%s name=%q)", + in.ObjToken, a.FileToken, a.Name) + continue + } + if !parseableAttachmentExts[ext] { + continue + } + data, derr := client.downloadMediaFile(ctx, a.FileToken) + if derr != nil { + logger.Warnf(ctx, "[Feishu] doc %s: attachment %q (token=%s) download failed: %v", + in.ObjToken, a.Name, a.FileToken, derr) + items = append(items, &types.FetchedItem{ + ExternalID: childID, + Title: a.Name, + SourceResourceID: in.ResourceID, + Metadata: FeishuErrorItemMeta(derr, childMeta()), + }) + continue + } + if len(data) < MinAttachmentBytes { + logger.Infof(ctx, "[Feishu] doc %s: skipping tiny attachment %q (token=%s, %d bytes < %d)", + in.ObjToken, a.Name, a.FileToken, len(data), MinAttachmentBytes) + continue + } + items = append(items, &types.FetchedItem{ + ExternalID: childID, + Title: a.Name, + Content: data, + ContentType: "application/octet-stream", + FileName: SanitizeFileName(a.Name), + URL: in.URL, + UpdatedAt: in.EditTime, + SourceResourceID: in.ResourceID, + Metadata: childMeta(), + }) + } + + imgMeta := func() map[string]string { + m := maps.Clone(in.BaseMeta) + m["parent_node_token"] = in.DocToken + m["embedded_image"] = "true" + return m + } + for _, b := range blocks { + if b.BlockType != BlockTypeImage || b.Image == nil || b.Image.Token == "" { + continue + } + childID := types.SubtreeChildID(in.DocToken, "image", b.Image.Token) + keep = append(keep, childID) // present in the doc → never sweep as stale + if !in.MultimodalEnabled { + continue // KB can't OCR images; the inline placeholder is all we keep + } + data, derr := client.downloadMediaFile(ctx, b.Image.Token) + if derr != nil { + logger.Warnf(ctx, "[Feishu] doc %s: image (token=%s) download failed: %v", + in.ObjToken, b.Image.Token, derr) + items = append(items, &types.FetchedItem{ + ExternalID: childID, + Title: fmt.Sprintf("%s(内嵌图片)", in.Title), + SourceResourceID: in.ResourceID, + Metadata: FeishuErrorItemMeta(derr, imgMeta()), + }) + continue + } + if len(data) < MinAttachmentBytes { + continue // decorative micro-image (icon/spacer) + } + ext, contentType, ok := SupportedImageExt(data) + if !ok { + logger.Warnf(ctx, "[Feishu] doc %s: skipping image (token=%s) of unsupported type %q", + in.ObjToken, b.Image.Token, contentType) + continue + } + items = append(items, &types.FetchedItem{ + ExternalID: childID, + Title: fmt.Sprintf("%s(内嵌图片)", in.Title), + Content: data, + ContentType: contentType, + FileName: "image-" + b.Image.Token + ext, + URL: in.URL, + UpdatedAt: in.EditTime, + SourceResourceID: in.ResourceID, + Metadata: imgMeta(), + }) + } + + main.SubtreeKeep = keep + return items, nil +} + +// exportDocxFallback exports a docx document via the async export API and +// returns a single FetchedItem containing the exported .docx binary. Used by +// FetchDocxWithBlocks when the blocks API is unavailable or renders empty. +func exportDocxFallback(ctx context.Context, client *Client, in DocxFetchInput) (*types.FetchedItem, error) { + data, fileName, err := client.ExportAndDownload(ctx, in.ObjToken, "docx") + if err != nil { + return nil, fmt.Errorf("export %s (docx): %w", in.Title, err) + } + + ext := ExportFileExtToSuffix[ObjTypeToExportFileExtension["docx"]] + if fileName == "" { + fileName = SanitizeFileName(in.Title) + ext + } else if !strings.HasSuffix(strings.ToLower(fileName), ext) { + fileName = SanitizeFileName(fileName) + ext + } + + return &types.FetchedItem{ + ExternalID: in.DocToken, + Title: in.Title, + Content: data, + ContentType: "application/octet-stream", + FileName: fileName, + URL: in.URL, + UpdatedAt: in.EditTime, + SourceResourceID: in.ResourceID, + Metadata: in.BaseMeta, + }, nil +} diff --git a/internal/datasource/connector/feishu/tally_test.go b/internal/datasource/connector/feishu/core/tally_test.go similarity index 90% rename from internal/datasource/connector/feishu/tally_test.go rename to internal/datasource/connector/feishu/core/tally_test.go index 04c3445a6..430e984b3 100644 --- a/internal/datasource/connector/feishu/tally_test.go +++ b/internal/datasource/connector/feishu/core/tally_test.go @@ -1,4 +1,4 @@ -package feishu +package core import ( "strings" @@ -10,9 +10,9 @@ func TestFetchTally_CountsAndSummary(t *testing.T) { tally.fetch() tally.fetch() tally.fetch() - tally.skip("mindnote") - tally.skip("mindnote") - tally.skip("slides") + tally.Skip("mindnote") + tally.Skip("mindnote") + tally.Skip("slides") tally.fail() if got := tally.skipped(); got != 3 { diff --git a/internal/datasource/connector/feishu/core/types.go b/internal/datasource/connector/feishu/core/types.go new file mode 100644 index 000000000..997d6f9ef --- /dev/null +++ b/internal/datasource/connector/feishu/core/types.go @@ -0,0 +1,359 @@ +// Package feishu implements the Feishu (飞书/Lark) data source connector for WeKnora. +// +// It syncs documents from Feishu Wiki spaces and cloud documents into WeKnora knowledge bases. +// +// Feishu API docs: +// - Wiki spaces: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space/list +// - Wiki nodes: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space-node/list +// - Export tasks: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/export-user-guide +// - File download: https://open.feishu.cn/document/server-docs/docs/drive-v1/file/download +// - Auth: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal +package core + +import ( + "strings" + "time" +) + +// Config holds Feishu-specific configuration for the data source connector. +// Uses the self-built app (企业自建应用) authentication model. +type Config struct { + // App ID from Feishu developer console + AppID string `json:"app_id"` + + // App Secret from Feishu developer console + AppSecret string `json:"app_secret"` + + // Base URL for Feishu API (default: https://open.feishu.cn) + // Use https://open.larksuite.com for Lark (international) deployments + BaseURL string `json:"base_url,omitempty"` + + // Timezone is the IANA name (e.g. "Asia/Shanghai", "America/New_York") used to + // render bitable date cells. Feishu stores a date as a UTC instant, but the + // calendar date a user sees is that instant in the table's timezone, so + // rendering in UTC shifts the date. Empty defaults to GMT+8, which matches + // Feishu (mainland) tenants; set it for Lark tenants in other zones. + Timezone string `json:"timezone,omitempty"` +} + +// defaultTimezoneOffsetSeconds is GMT+8 (the Feishu mainland default), used when +// no timezone is configured. A fixed zone avoids any dependency on system tzdata, +// which minimal container images may omit. +const defaultTimezoneOffsetSeconds = 8 * 3600 + +// resolveLocation returns the *time.Location used to render bitable date cells. +// An empty name yields a fixed GMT+8 zone (no tzdata dependency); a named zone is +// loaded from the system tz database, falling back to GMT+8 if it is unavailable. +func resolveLocation(name string) *time.Location { + if name != "" { + if loc, err := time.LoadLocation(name); err == nil { + return loc + } + } + return time.FixedZone("GMT+8", defaultTimezoneOffsetSeconds) +} + +// defaultBaseURL is the default Feishu Open Platform API base URL. +const defaultBaseURL = "https://open.feishu.cn" + +// larkBaseURL is the Lark (international) API base URL. +const larkBaseURL = "https://open.larksuite.com" + +// GetBaseURL returns the effective base URL, defaulting to Feishu if not set. +func (c *Config) GetBaseURL() string { + if c.BaseURL != "" { + return c.BaseURL + } + return defaultBaseURL +} + +// --- Export format constants --- +// Used by the export task API: POST /drive/v1/export_tasks + +const ( + // ExportTypeDocx exports Feishu documents to .docx format. + ExportTypeDocx = "docx" + // ExportTypeXlsx exports spreadsheets / bitable to .xlsx format. + ExportTypeXlsx = "xlsx" + // ExportTypePDF exports documents to .pdf format (fallback). + ExportTypePDF = "pdf" +) + +// ObjTypeToExportFileExtension maps Feishu obj_type to the best export file_extension. +var ObjTypeToExportFileExtension = map[string]string{ + "docx": ExportTypeDocx, + "doc": ExportTypeDocx, + "sheet": ExportTypeXlsx, + "bitable": ExportTypeXlsx, +} + +// ObjTypeToExportType maps Feishu obj_type to the export API "type" parameter. +// See: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/create +var ObjTypeToExportType = map[string]string{ + "docx": "docx", + "doc": "doc", + "sheet": "sheet", + "bitable": "bitable", +} + +// ExportFileExtToSuffix maps export file_extension to the file suffix for FileName. +var ExportFileExtToSuffix = map[string]string{ + ExportTypeDocx: ".docx", + ExportTypeXlsx: ".xlsx", + ExportTypePDF: ".pdf", +} + +// --- Feishu API response structures --- + +// ApiResponse is the common Feishu API response wrapper. +type ApiResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +// TokenResponse is the response for tenant_access_token API. +type TokenResponse struct { + ApiResponse + TenantAccessToken string `json:"tenant_access_token"` + Expire int `json:"expire"` // seconds +} + +// WikiSpaceListData is the data payload of WikiSpaceListResponse. +type WikiSpaceListData struct { + Items []WikiSpace `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` +} + +// WikiSpaceListResponse is the response for GET /open-apis/wiki/v2/spaces. +type WikiSpaceListResponse struct { + ApiResponse + Data WikiSpaceListData `json:"data"` +} + +// WikiSpace represents a Feishu Wiki space. +type WikiSpace struct { + SpaceID string `json:"space_id"` + Name string `json:"name"` + Description string `json:"description"` + Visibility string `json:"visibility"` // "public" or "private" +} + +// WikiNodeListData is the data payload of WikiNodeListResponse. +type WikiNodeListData struct { + Items []WikiNode `json:"items"` + HasMore bool `json:"has_more"` + PageToken string `json:"page_token"` +} + +// WikiNodeListResponse is the response for GET /open-apis/wiki/v2/spaces/:space_id/nodes. +type WikiNodeListResponse struct { + ApiResponse + Data WikiNodeListData `json:"data"` +} + +// WikiNode represents a node (document or folder) in a Feishu Wiki space. +type WikiNode struct { + SpaceID string `json:"space_id"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` // document token + ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides" + ParentNodeID string `json:"parent_node_token"` + NodeType string `json:"node_type"` // "origin" or "shortcut" + OriginNodeID string `json:"origin_node_id"` + OriginSpaceID string `json:"origin_space_id"` + HasChild bool `json:"has_child"` + Title string `json:"title"` + Creator string `json:"creator"` + Owner string `json:"owner"` + ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string) + ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes + NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string) + NodeEditTime string `json:"node_edit_time"` // node edit time (unix timestamp string) — only tracks node attribute changes +} + +// WikiNodeInfoData is the data payload of WikiNodeInfoResponse. +type WikiNodeInfoData struct { + Node WikiNode `json:"node"` +} + +// WikiNodeInfoResponse is the response for GET /open-apis/wiki/v2/spaces/get_node. +type WikiNodeInfoResponse struct { + ApiResponse + Data WikiNodeInfoData `json:"data"` +} + +// --- Export task API responses --- + +// docRawContentData is the data payload of docRawContentResponse. +type docRawContentData struct { + Content string `json:"content"` +} + +// docRawContentResponse is the response for GET /open-apis/docx/v1/documents/:document_id/raw_content. +// Deprecated: prefer export API for full-fidelity document export. +type docRawContentResponse struct { + ApiResponse + Data docRawContentData `json:"data"` +} + +// ExportTaskCreateData is the data payload of ExportTaskCreateResponse. +type ExportTaskCreateData struct { + Ticket string `json:"ticket"` +} + +// ExportTaskCreateResponse is the response for POST /drive/v1/export_tasks. +type ExportTaskCreateResponse struct { + ApiResponse + Data ExportTaskCreateData `json:"data"` +} + +// ExportTaskResult is the per-task result inside ExportTaskStatusData. +type ExportTaskResult struct { + FileToken string `json:"file_token"` + FileSize int64 `json:"file_size"` + // JobStatus: 0=success, 1=initializing, 2=processing + JobStatus int `json:"job_status"` + JobErrorMsg string `json:"job_error_msg"` + FileName string `json:"file_name"` +} + +// ExportTaskStatusData is the data payload of ExportTaskStatusResponse. +type ExportTaskStatusData struct { + Result ExportTaskResult `json:"result"` +} + +// ExportTaskStatusResponse is the response for GET /drive/v1/export_tasks/{ticket}. +type ExportTaskStatusResponse struct { + ApiResponse + Data ExportTaskStatusData `json:"data"` +} + +// --- File download response --- + +// driveFileMeta is one entry of driveFileMetaData.Metas. +type driveFileMeta struct { + DocToken string `json:"doc_token"` + DocType string `json:"doc_type"` + Title string `json:"title"` +} + +// driveFileMetaData is the data payload of driveFileMetaResponse. +type driveFileMetaData struct { + Metas []driveFileMeta `json:"metas"` +} + +// driveFileMetaResponse is the response for GET /drive/v1/metas for file type nodes. +type driveFileMetaResponse struct { + ApiResponse + Data driveFileMetaData `json:"data"` +} + +// FeishuCursor stores incremental sync state for Feishu. +type FeishuCursor struct { + // LastSyncTime is the timestamp of the last successful sync. + LastSyncTime time.Time `json:"last_sync_time"` + + // SpaceNodeTimes maps space_id -> node_token -> last known edit time. + // Used to detect which nodes have changed since last sync. + SpaceNodeTimes map[string]map[string]string `json:"space_node_times,omitempty"` +} + +// --- Drive (云盘) file listing types (feishu_drive / lark_drive connectors) --- +// Added by feat/datasource-feishu-drive. These are independent of the wiki +// types above and do not affect the wiki connector. + +// DriveFile represents a file/folder in Feishu Drive (云空间). Returned by +// GET /open-apis/drive/v1/files?folder_token=xxx. The list API returns +// modified_time directly (verified), so no batch_query/metas call is needed for +// incremental detection - see ADR-0002. +type DriveFile struct { + Token string `json:"token"` + Name string `json:"name"` + Type string `json:"type"` // doc/docx/sheet/bitable/file/folder/shortcut/mindnote/slides/board + ParentToken string `json:"parent_token"` + URL string `json:"url"` + CreatedTime string `json:"created_time"` // unix seconds string + ModifiedTime string `json:"modified_time"` // unix seconds string - 等价知识库 obj_edit_time + OwnerID string `json:"owner_id"` + // ShortcutInfo is populated only for type=="shortcut". target_type can only + // be doc/sheet/mindnote/bitable/file/docx (Feishu does not allow shortcuts to + // folders, verified) - see ADR-0002 / glossary shortcut entry. + ShortcutInfo *driveShortcutInfo `json:"shortcut_info,omitempty"` +} + +// driveShortcutInfo is the target metadata of a Drive shortcut. +type driveShortcutInfo struct { + TargetToken string `json:"target_token"` + TargetType string `json:"target_type"` +} + +// DriveFileListData is the data payload of DriveFileListResponse. +type DriveFileListData struct { + Files []DriveFile `json:"files"` + HasMore bool `json:"has_more"` + NextPageToken string `json:"next_page_token"` +} + +// DriveFileListResponse is the response for GET /open-apis/drive/v1/files. +type DriveFileListResponse struct { + ApiResponse + Data DriveFileListData `json:"data"` +} + +// driveFolderMetaData is the data payload of driveFolderMetaResponse. +type driveFolderMetaData struct { + ID string `json:"id"` + Name string `json:"name"` + Token string `json:"token"` + CreateUid string `json:"createUid"` + EditUid string `json:"editUid"` + ParentID string `json:"parentId"` + OwnUid string `json:"ownUid"` +} + +// driveFolderMetaResponse is the response for GET /open-apis/drive/explorer/v2/folder/:folderToken/meta. +// Used to resolve a root folder's human-readable name (the list API only returns +// the folder's children, not the folder itself). +type driveFolderMetaResponse struct { + ApiResponse + Data driveFolderMetaData `json:"data"` +} + +// DriveFileListFailure records a single sub-folder listing that failed during a +// recursive walk. Mirrors WikiNodeListFailure. +type DriveFileListFailure struct { + FolderToken string + Err error +} + +// PartialDriveFileListError aggregates per-folder listing failures so the walk +// can continue and the caller can still surface the partial result. Mirrors +// PartialWikiNodeListError. +type PartialDriveFileListError struct { + Failures []DriveFileListFailure +} + +func (e *PartialDriveFileListError) Error() string { + if e == nil || len(e.Failures) == 0 { + return "partial drive file listing failed" + } + parts := make([]string, 0, len(e.Failures)) + for _, failure := range e.Failures { + parts = append(parts, failure.Err.Error()) + } + return strings.Join(parts, "; ") +} + +// FeishuDriveCursor stores incremental sync state for Feishu Drive (云盘). +// Structurally symmetric with FeishuCursor: outer key = resourceID +// ("folderToken" or "folderToken:fileToken"), inner key = file_token, +// value = modified_time. See ADR-0001. +type FeishuDriveCursor struct { + // LastSyncTime is the timestamp of the last successful sync. + LastSyncTime time.Time `json:"last_sync_time"` + + // FileTimes maps resourceID -> file_token -> last known modified_time. + // Used to detect which files have changed since last sync. + FileTimes map[string]map[string]string `json:"file_times,omitempty"` +} diff --git a/internal/datasource/connector/feishu/drive/connector.go b/internal/datasource/connector/feishu/drive/connector.go new file mode 100644 index 000000000..1e2d24327 --- /dev/null +++ b/internal/datasource/connector/feishu/drive/connector.go @@ -0,0 +1,615 @@ +package drive + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// DriveConnector implements the datasource.Connector (and StreamingConnector) +// interface for Feishu/Lark Drive (云盘) mode. It shares core.Client/core.Config/core.Region +// and the export/download logic with the wiki Connector; only resource +// enumeration and Fetch dispatch differ. See 飞书云盘数据源设计.md and +// ADR-0001..0004. +type DriveConnector struct { + region core.Region +} + +// NewDriveConnector creates a Drive connector for the given region +// (RegionFeishuDrive or RegionLarkDrive). +func NewDriveConnector(region core.Region) *DriveConnector { + return &DriveConnector{region: region} +} + +// Drive supports resumable streaming sync; the service prefers FetchStream over +// FetchAll/FetchIncremental when a connector implements StreamingConnector. +var _ datasource.StreamingConnector = (*DriveConnector)(nil) + +// Type returns the connector type identifier. +func (c *DriveConnector) Type() string { + return c.region.ConnectorType +} + +// Validate verifies that the Drive configuration is valid by testing +// connectivity. It does not validate folder_token here - that is done in +// ListResources when the user loads the tree root. Mirrors the wiki Connector. +func (c *DriveConnector) Validate(ctx context.Context, config *types.DataSourceConfig) error { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return err + } + + client := core.NewClient(feishuConfig) + if err := client.Ping(ctx); err != nil { + return fmt.Errorf("%s connection failed: %w", c.region.Label, err) + } + return nil +} + +// ListResources lists Feishu Drive resources for selection, loading the tree +// lazily one level at a time. Mirrors the wiki Connector.ListResources shape +// but with the Drive difference that the root is user-supplied (config. +// ResourceIDs[0]) rather than enumerated via ListWikiSpaces. +// +// - parentID == "" -> return the user-supplied root +// folder (from config.ResourceIDs[0]) as the single root resource +// (HasChildren=true). Drive has no "space list" API, so the root is +// user-supplied. folder_token == "" is rejected (ADR-0004). +// - parentID == folderToken -> ListDriveFiles(folderToken) +// returns the direct children. +// - parentID == "folderToken:subFolderToken" -> ListDriveFiles(subFolderToken) +// returns that sub-folder's direct children. +// +// Each core.DriveFile becomes a Resource: folder HasChildren=true, others false. +// resourceID encoding: root = folderToken; child = folderToken + ":" + fileToken +// (reuses core.FeishuWikiNodeResourceSeparator). See ADR-0001 §3.4. +func (c *DriveConnector) ListResources( + ctx context.Context, config *types.DataSourceConfig, parentID string, +) ([]types.Resource, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + + client := core.NewClient(feishuConfig) + + if parentID == "" { + // Root load: read the user-supplied folder_token from config.ResourceIDs. + rootFolderToken := driveRootFolderToken(config) + if rootFolderToken == "" { + return nil, fmt.Errorf("folder_token is required; specify a Drive folder token") + } + // Validate access by listing the root's direct children (also lazy-loads + // the first level for the picker). Reuse the list call rather than a + // separate ping. + files, err := client.ListDriveFilesAllPages(ctx, rootFolderToken) + if err != nil { + return nil, fmt.Errorf("list feishu drive folder %s: %w", rootFolderToken, err) + } + _ = files // children returned via the parentID == rootFolderToken branch below + // Resolve the root folder's human-readable name via the folder meta API. + // Best-effort: on failure (no permission / not found) fall back to the + // folder_token so the picker still renders something usable. + folderName := rootFolderToken + if meta, mErr := client.GetDriveFolderMeta(ctx, rootFolderToken); mErr != nil { + logger.Warnf(ctx, "[FeishuDrive] resolve root folder name failed: %v (falling back to token)", mErr) + } else if meta.Data.Name != "" { + folderName = meta.Data.Name + } + return []types.Resource{c.driveFolderToResource(rootFolderToken, "", rootFolderToken, folderName)}, nil + } + + // Lazy load: list only the direct children of the given folder. + rootFolderToken, folderToken := parseDriveResourceID(parentID) + if folderToken == "" { + // parentID is a bare root folder token -> list its children. + folderToken = rootFolderToken + } + files, err := client.ListDriveFilesAllPages(ctx, folderToken) + if err != nil { + return nil, fmt.Errorf("list feishu drive files under %s: %w", parentID, err) + } + + resources := make([]types.Resource, 0, len(files)) + for _, f := range files { + resources = append(resources, c.driveFileToResource(rootFolderToken, f)) + } + return resources, nil +} + +// ResolveResourceAncestors returns the resource IDs of every parent folder that +// has to be expanded so the lazily-loaded picker can reveal each selection. +// +// The wiki connector walks up via GetWikiNode (parent_node_token) in O(depth) +// single-node queries. Drive has no single-file parent query API (verified - +// metas/batch_query does not return parent), so we walk top-down from the root +// folder with ListDriveFiles and share the traversal across all selections in +// the same root. Best-effort: a broken path just stays collapsed. See ADR-0003. +func (c *DriveConnector) ResolveResourceAncestors( + ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, +) ([]string, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + client := core.NewClient(feishuConfig) + + seen := make(map[string]bool) + ancestors := make([]string, 0) + add := func(id string) { + if id != "" && !seen[id] { + seen[id] = true + ancestors = append(ancestors, id) + } + } + + // Group selections by root folder so one shared traversal covers them all. + type selection struct { + fileToken string + } + rootSelections := make(map[string][]selection) + for _, rid := range resourceIDs { + rootFolderToken, fileToken := parseDriveResourceID(rid) + if fileToken == "" { + // A root-level selection is already a top-level node; nothing to reveal. + continue + } + add(rootFolderToken) + rootSelections[rootFolderToken] = append(rootSelections[rootFolderToken], selection{fileToken}) + } + + // For each root, BFS from the root: at each folder, ListDriveFiles and check + // which selections are direct children (record their parent chain) and which + // sub-folders may still contain selections (enqueue). Shared traversal means + // selections in the same subtree reuse list calls. + for rootFolderToken, sels := range rootSelections { + remaining := make(map[string]bool, len(sels)) + for _, s := range sels { + remaining[s.fileToken] = true + } + // parentChain[fileToken] = resourceID of its parent folder + parentChain := make(map[string]string) + + queue := []string{rootFolderToken} + for len(queue) > 0 && len(remaining) > 0 { + cur := queue[0] + queue = queue[1:] + + files, err := client.ListDriveFilesAllPages(ctx, cur) + if err != nil { + logger.Warnf(ctx, "[FeishuDrive] resolve ancestors: list %s: %v", cur, err) + break // best-effort: stop this root's traversal + } + for _, f := range files { + if remaining[f.Token] { + delete(remaining, f.Token) + // Record the parent chain from root down to this file's parent. + chain := buildDriveAncestorChain(rootFolderToken, cur, parentChain) + for _, a := range chain { + add(a) + } + } + if f.Type == "folder" { + parentChain[f.Token] = makeDriveResourceID(rootFolderToken, cur) + queue = append(queue, f.Token) + } + } + } + } + + return ancestors, nil +} + +// buildDriveAncestorChain walks the parentChain map from cur up to root, +// returning the resourceIDs (root, ... , cur's parent) in root-first order. +func buildDriveAncestorChain(rootFolderToken, cur string, parentChain map[string]string) []string { + var chain []string + node := cur + for node != "" && node != rootFolderToken { + parent, ok := parentChain[node] + if !ok { + break + } + chain = append([]string{parent}, chain...) + _, parentFolderToken := parseDriveResourceID(parent) + node = parentFolderToken + } + return chain +} + +// FetchAll performs a full sync of all documents from the selected Drive +// folders. Defensive fallback path - the service prefers FetchStream when the +// connector implements StreamingConnector. +func (c *DriveConnector) FetchAll( + ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, +) ([]types.FetchedItem, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + client := core.NewClient(feishuConfig) + return core.FetchAllEngine(ctx, client, config, resourceIDs, driveOps{region: c.region}) +} + +// FetchIncremental performs an incremental sync by comparing file modified_time +// against the previously recorded state. Defensive fallback path - the service +// prefers FetchStream. Routed through the same engine as FetchStream, so the +// #2136 failure-doesn't-advance-cursor semantics apply here too. +func (c *DriveConnector) FetchIncremental( + ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor, +) ([]types.FetchedItem, *types.SyncCursor, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, nil, err + } + client := core.NewClient(feishuConfig) + ops := driveOps{region: c.region} + if len(config.ResourceIDs) == 0 { + return nil, nil, errors.New(ops.EmptyResourceIDsError()) + } + return core.FetchIncrementalEngine(ctx, client, config, cursor, ops) +} + +// FetchStream performs a resumable, memory-bounded sync. It unifies the full +// and incremental paths: with cursor == nil it fetches everything, and with a +// cursor it skips files whose recorded modified_time is unchanged - the same +// mechanism that lets a sync which timed out mid-traversal resume from the last +// checkpoint instead of restarting (Tencent/WeKnora#2136). +// +// The per-node loop lives in the shared engine (engine.go); this shell only +// wires the Drive NodeOps adapter. +func (c *DriveConnector) FetchStream( + ctx context.Context, config *types.DataSourceConfig, + cursor *types.SyncCursor, h datasource.StreamHandler, +) (*types.SyncCursor, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + client := core.NewClient(feishuConfig) + ops := driveOps{region: c.region} + if len(config.ResourceIDs) == 0 { + return nil, errors.New(ops.EmptyResourceIDsError()) + } + return core.FetchStreamEngine(ctx, client, config, cursor, h, ops) +} + +// driveOps adapts the Drive DriveConnector to the generic sync engine. It +// carries the region (for channel + URL) and encodes/decodes the Drive cursor +// wire format (core.FeishuDriveCursor / file_times) so the engine stays format-agnostic. +type driveOps struct { + region core.Region +} + +func (o driveOps) List(ctx context.Context, client *core.Client, resourceID string) ([]core.DriveFile, error, error) { + files, err := listDriveFilesForResource(ctx, client, resourceID) + if err == nil { + return files, nil, nil + } + var partial *core.PartialDriveFileListError + if errors.As(err, &partial) { + return files, err, nil + } + return files, nil, err +} + +func (o driveOps) Token(n core.DriveFile) string { return n.Token } +func (o driveOps) Title(n core.DriveFile) string { return n.Name } +func (o driveOps) ObjType(n core.DriveFile) string { return n.Type } +func (o driveOps) EditTime(n core.DriveFile) string { return n.ModifiedTime } + +func (o driveOps) Fetch(ctx context.Context, client *core.Client, n core.DriveFile, resourceID string, multimodal bool) ([]*types.FetchedItem, error) { + return fetchDriveFileContent(ctx, client, n, resourceID, multimodal, o.region) +} + +func (o driveOps) ListFailureItems(resourceID string, partial error) []types.FetchedItem { + var pe *core.PartialDriveFileListError + if errors.As(partial, &pe) { + return appendDriveFileListFailureItems(nil, resourceID, o.channel(), pe.Failures) + } + return nil +} + +func (o driveOps) channel() string { + if o.region.ConnectorType == types.ConnectorTypeLarkDrive { + return types.ChannelLarkDrive + } + return types.ChannelFeishuDrive +} + +func (o driveOps) ResourceNoun() string { return "files" } +func (o driveOps) EmptyResourceIDsError() string { + return "no resource IDs (Drive folder tokens) configured" +} +func (o driveOps) LogTag() string { return "[FeishuDrive]" } + +func (o driveOps) DecodeCursorTimes(m map[string]interface{}) map[string]map[string]string { + var prev core.FeishuDriveCursor + b, _ := json.Marshal(m) + _ = json.Unmarshal(b, &prev) + return prev.FileTimes +} + +func (o driveOps) EncodeCursor(times map[string]map[string]string, lastSync time.Time) *types.SyncCursor { + fc := core.FeishuDriveCursor{LastSyncTime: lastSync, FileTimes: times} + m := make(map[string]interface{}) + b, _ := json.Marshal(fc) + _ = json.Unmarshal(b, &m) + return &types.SyncCursor{LastSyncTime: lastSync, ConnectorCursor: m} +} + +// fetchDriveFileContent fetches the content of a single Drive file and converts +// it to FetchedItems. Dispatches by file.Type, mirroring the wiki +// fetchNodeContent. Shortcuts have already been expanded to their target by +// ListDriveFilesRecursiveFrom, so this only sees the target type. +// +// - docx -> blocks API (Markdown) with export fallback; may return attachments/images +// - doc/sheet/bitable -> ExportAndDownload -> docx/xlsx +// - file -> DownloadDriveFile -> original file +// - mindnote/slides/board -> Skip (no API), returns (nil, nil) +func fetchDriveFileContent( + ctx context.Context, client *core.Client, file core.DriveFile, resourceID string, multimodalEnabled bool, region core.Region, +) ([]*types.FetchedItem, error) { + if !core.IsSupportedDocType(file.Type) { + return nil, nil + } + + editTime := core.ParseFeishuTimestamp(file.ModifiedTime) + // Channel marks the knowledge "source" label. Drive uses its own channel + // (feishu_drive / lark_drive) so Drive docs show "飞书云盘" / "Lark 云盘" + // distinct from the wiki connector's "飞书". + channel := types.ChannelFeishuDrive + if region.ConnectorType == types.ConnectorTypeLarkDrive { + channel = types.ChannelLarkDrive + } + baseMeta := map[string]string{ + "obj_token": file.Token, + "obj_type": file.Type, + "file_token": file.Token, + "folder_token": file.ParentToken, + "channel": channel, + } + + switch file.Type { + case "docx": + return core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ + DocToken: file.Token, + ObjToken: file.Token, + Title: file.Name, + URL: file.URL, + ResourceID: resourceID, + EditTime: editTime, + BaseMeta: baseMeta, + MultimodalEnabled: multimodalEnabled, + }) + + case "doc", "sheet", "bitable": + data, fileName, err := client.ExportAndDownload(ctx, file.Token, file.Type) + if err != nil { + return nil, fmt.Errorf("export %s (%s): %w", file.Name, file.Type, err) + } + + ext := core.ExportFileExtToSuffix[core.ObjTypeToExportFileExtension[file.Type]] + if fileName == "" { + fileName = core.SanitizeFileName(file.Name) + ext + } else if !strings.HasSuffix(strings.ToLower(fileName), ext) { + fileName = core.SanitizeFileName(fileName) + ext + } + + return []*types.FetchedItem{{ + ExternalID: file.Token, + Title: file.Name, + Content: data, + ContentType: "application/octet-stream", + FileName: fileName, + URL: file.URL, + UpdatedAt: editTime, + SourceResourceID: resourceID, + Metadata: baseMeta, + }}, nil + + case "file": + data, err := client.DownloadDriveFile(ctx, file.Token) + if err != nil { + return nil, fmt.Errorf("download file %s (%s): %w", file.Name, file.Token, err) + } + + fileName := file.Name + if fileName == "" { + fileName = file.Token + } + + return []*types.FetchedItem{{ + ExternalID: file.Token, + Title: file.Name, + Content: data, + ContentType: "application/octet-stream", + FileName: fileName, + URL: file.URL, + UpdatedAt: editTime, + SourceResourceID: resourceID, + Metadata: baseMeta, + }}, nil + + default: + return nil, nil + } +} + +// --- Helpers --- + +// makeDriveResourceID encodes a Drive ResourceID: "folderToken" (root) or +// "folderToken:fileToken" (child). Reuses core.FeishuWikiNodeResourceSeparator. +func makeDriveResourceID(rootFolderToken, fileToken string) string { + if fileToken == "" { + return rootFolderToken + } + return rootFolderToken + core.FeishuWikiNodeResourceSeparator + fileToken +} + +// parseDriveResourceID splits a Drive resourceID into (rootFolderToken, fileToken). +// Mirrors parseWikiResourceID. +func parseDriveResourceID(resourceID string) (rootFolderToken, fileToken string) { + rootFolderToken, fileToken, _ = strings.Cut(resourceID, core.FeishuWikiNodeResourceSeparator) + return rootFolderToken, fileToken +} + +// listDriveFilesForResource lists the files to sync for a given resourceID. +// A resourceID is either a bare root folderToken (sync the whole subtree) or +// "rootFolderToken:fileToken" (sync a single selected file or sub-folder). +// +// For a single-file selection we cannot pass the fileToken to +// ListDriveFilesRecursiveFrom - that API expects a folder and returns 1061002 +// (params error) for a file token. Instead we walk the root folder subtree (the +// file's parent) and filter to just the selected fileToken. This mirrors the +// wiki connector, which resolves a single selected node via GetWikiNode; Drive +// has no single-file meta API, so filtering the subtree walk is the equivalent. +// +// A sub-folder selection (fileToken is itself a folder) is handled by walking +// that sub-folder's subtree directly - ListDriveFilesRecursiveFrom accepts a +// folder token, so no filtering is needed there. +func listDriveFilesForResource( + ctx context.Context, client *core.Client, resourceID string, +) ([]core.DriveFile, error) { + rootFolderToken, fileToken := parseDriveResourceID(resourceID) + if fileToken == "" { + return client.ListDriveFilesRecursiveFrom(ctx, rootFolderToken) + } + files, err := client.ListDriveFilesRecursiveFrom(ctx, fileToken) + if err == nil { + return files, nil + } + if !isDriveNotFolderError(err) { + return nil, err + } + all, walkErr := client.ListDriveFilesRecursiveFrom(ctx, rootFolderToken) + if walkErr != nil { + var partialErr *core.PartialDriveFileListError + if !errors.As(walkErr, &partialErr) { + return nil, walkErr + } + all = filterDriveFileByToken(all, fileToken) + if len(all) == 0 { + return nil, walkErr + } + return all, walkErr + } + return filterDriveFileByToken(all, fileToken), nil +} + +// isDriveNotFolderError reports whether err indicates the token was not a +// folder (1061002 params error from the list API when a file token is passed). +func isDriveNotFolderError(err error) bool { + s := strings.ToLower(err.Error()) + return strings.Contains(s, "1061002") || strings.Contains(s, "params error") +} + +// filterDriveFileByToken returns only the entries whose Token matches token. +func filterDriveFileByToken(files []core.DriveFile, token string) []core.DriveFile { + var out []core.DriveFile + for _, f := range files { + if f.Token == token { + out = append(out, f) + } + } + return out +} + +// driveRootFolderToken extracts the user-supplied root folder_token from the +// data source config (ResourceIDs[0]). +func driveRootFolderToken(config *types.DataSourceConfig) string { + if config == nil || len(config.ResourceIDs) == 0 { + return "" + } + root, _ := parseDriveResourceID(config.ResourceIDs[0]) + return root +} + +// driveFolderToResource builds the root Resource for a Drive folder. The root +// folder's name is resolved via GetDriveFolderMeta by the caller (best-effort, +// falling back to the token). For sub-folders, use driveFileToResource instead - +// the list API returns each child folder's Name. +// +// The root folder's ExternalID is the bare rootFolderToken (no ":fileToken" +// suffix) so it matches the resource_id the user saved in +// form.config.resource_ids = [folderToken]. A "token:token" encoding would +// break selection matching on edit. +func (c *DriveConnector) driveFolderToResource(rootFolderToken, parentToken, folderToken, name string) types.Resource { + if name == "" { + name = folderToken + } + return types.Resource{ + ExternalID: rootFolderToken, + Name: name, + Type: "drive_folder", + URL: c.region.DriveFolderURL(folderToken), + HasChildren: true, + Metadata: map[string]interface{}{ + "folder_token": folderToken, + }, + } +} + +// driveFileToResource converts a core.DriveFile (list result) into a picker Resource. +// The ParentID must match the parent folder's ExternalID: the root folder's +// ExternalID is the bare rootFolderToken (see driveFolderToResource), while any +// sub-folder's ExternalID is "rootFolderToken:folderToken". Direct children of +// the root have file.ParentToken == rootFolderToken, so their ParentID is the +// bare rootFolderToken; deeper descendants use the encoded form. +func (c *DriveConnector) driveFileToResource(rootFolderToken string, file core.DriveFile) types.Resource { + name := file.Name + if name == "" { + name = file.Token + } + + modifiedAt := core.ParseFeishuTimestamp(file.ModifiedTime) + + parentID := makeDriveResourceID(rootFolderToken, file.ParentToken) + if file.ParentToken == rootFolderToken || file.ParentToken == "" { + // Direct child of the root folder: parent is the root, whose + // ExternalID is the bare rootFolderToken (no ":token" suffix). + parentID = rootFolderToken + } + + return types.Resource{ + ExternalID: makeDriveResourceID(rootFolderToken, file.Token), + Name: name, + Type: file.Type, + URL: file.URL, + ParentID: parentID, + HasChildren: file.Type == "folder", + ModifiedAt: modifiedAt, + Metadata: map[string]interface{}{ + "file_token": file.Token, + "obj_type": file.Type, + "folder_token": file.ParentToken, + }, + } +} + +// appendDriveFileListFailureItems converts Drive listing failures into error +// FetchedItems so the sync log surfaces which sub-folders could not be listed. +// Mirrors appendWikiNodeListFailureItems. +func appendDriveFileListFailureItems(items []types.FetchedItem, resourceID, channel string, failures []core.DriveFileListFailure) []types.FetchedItem { + for _, failure := range failures { + items = append(items, types.FetchedItem{ + ExternalID: failure.FolderToken, + Title: failure.FolderToken, + SourceResourceID: resourceID, + Metadata: core.FeishuErrorItemMeta(failure.Err, map[string]string{ + "channel": channel, + "folder_token": failure.FolderToken, + "failure_stage": "list_children", + }), + }) + } + return items +} diff --git a/internal/datasource/connector/feishu/drive/cursor_test.go b/internal/datasource/connector/feishu/drive/cursor_test.go new file mode 100644 index 000000000..76052726b --- /dev/null +++ b/internal/datasource/connector/feishu/drive/cursor_test.go @@ -0,0 +1,42 @@ +package drive + +import ( + "testing" + "time" + + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" +) + +// TestDriveCursorRoundTrip verifies driveOps.EncodeCursor/DecodeCursorTimes +// survive a JSON marshal/unmarshal cycle (snapshot isolation), mirroring the +// wiki TestFeishuCursorRoundTrip. Design §3.2: driveOps is new code that +// previously had no direct test. +func TestDriveCursorRoundTrip(t *testing.T) { + ops := driveOps{region: core.RegionFeishuDrive} + times := map[string]map[string]string{ + "folder1": {"fdoc1": "100", "fdoc2": "200"}, + "folder1:fdoc1": {"fdoc1": "100"}, + "folder2:subA": {"fdoc3": "300"}, + } + cur := ops.EncodeCursor(times, time.Unix(1000, 0)) + if cur == nil { + t.Fatal("EncodeCursor returned nil") + } + + got := ops.DecodeCursorTimes(cur.ConnectorCursor) + if len(got) != len(times) { + t.Fatalf("decoded %d resources, want %d", len(got), len(times)) + } + for rid, files := range times { + gotFiles, ok := got[rid] + if !ok { + t.Errorf("resource %q missing from decoded cursor", rid) + continue + } + for tok, mt := range files { + if gotFiles[tok] != mt { + t.Errorf("decoded[%q][%q] = %q, want %q", rid, tok, gotFiles[tok], mt) + } + } + } +} diff --git a/internal/datasource/connector/feishu/drive/drive_blocks_test.go b/internal/datasource/connector/feishu/drive/drive_blocks_test.go new file mode 100644 index 000000000..d057d3831 --- /dev/null +++ b/internal/datasource/connector/feishu/drive/drive_blocks_test.go @@ -0,0 +1,277 @@ +package drive + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" + "github.com/Tencent/WeKnora/internal/types" +) + +// makeDriveConfig builds a DataSourceConfig for the Drive connector, mirroring +// makeConfig but with the drive connector type and a multimodal toggle. +func makeDriveConfig(cfg *core.Config, resourceIDs []string, multimodal bool) *types.DataSourceConfig { + c := makeConfig(cfg, resourceIDs) + c.Type = types.ConnectorTypeFeishuDrive + c.MultimodalEnabled = multimodal + return c +} + +// fakeFeishuDriveDocx serves the Drive list endpoint plus the docx blocks / +// media / export endpoints. blocksMode controls the blocks API behaviour: +// - "ok": serve the given blocks +// - "fail": HTTP 500 (missing scope) → exercises the export fallback +// - "empty": serve only a page block → empty Markdown → export fallback +// +// The export trio is always registered so both fallback modes can complete. +func fakeFeishuDriveDocx(t *testing.T, files []core.DriveFile, docToken string, + blocks []core.DocxBlock, blocksMode string, mediaContent []byte, +) (*httptest.Server, *core.Config) { + t.Helper() + mux := http.NewServeMux() + + mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) + }) + + // Drive file listing (single page). + mux.HandleFunc("/open-apis/drive/v1/files", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, core.DriveFileListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.DriveFileListData{Files: files}, + }) + }) + + // Blocks API for the given docx document. + blocksPath := "/open-apis/docx/v1/documents/" + docToken + "/blocks" + switch blocksMode { + case "fail": + mux.HandleFunc(blocksPath, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"code":99991400,"msg":"insufficient scope"}`)) + }) + case "empty": + mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, core.DocxBlocksResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.DocxBlocksData{Items: []core.DocxBlock{{BlockID: "b1", BlockType: core.BlockTypePage}}}, + }) + }) + default: // "ok" + mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, core.DocxBlocksResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.DocxBlocksData{Items: blocks}, + }) + }) + } + + // Media download for File/Image block tokens. + mux.HandleFunc("/open-apis/drive/v1/medias/", func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/download") { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(mediaContent) + return + } + http.NotFound(w, r) + }) + + // Export trio for the fallback path. + mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, core.ExportTaskCreateResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskCreateData{Ticket: "ticket-drv"}, + }) + }) + mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-drv", func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, core.ExportTaskStatusResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskStatusData{ + Result: core.ExportTaskResult{FileToken: "ft-export-drv", FileSize: 512, JobStatus: 0, FileName: "drive-fallback.docx"}, + }, + }) + }) + mux.HandleFunc("/open-apis/drive/v1/export_tasks/file/ft-export-drv/download", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write([]byte("fake-drive-export-binary")) + }) + + ts := httptest.NewServer(mux) + t.Cleanup(ts.Close) + return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} +} + +func driveDocxBlocks(attToken, attName string) []core.DocxBlock { + return []core.DocxBlock{ + {BlockID: "b1", BlockType: core.BlockTypePage}, + {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ + Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "Hello drive"}}}, + }}, + {BlockID: "b3", BlockType: core.BlockTypeFile, File: &core.BlockFileRef{Token: attToken, Name: attName}}, + } +} + +const driveDocxFileToken = "fdoc1" + +func driveDocxFile() core.DriveFile { + return core.DriveFile{ + Token: driveDocxFileToken, + Type: "docx", + Name: "Drive Doc", + ModifiedTime: "500", + ParentToken: "folder1", + URL: "https://example.feishu.cn/file/" + driveDocxFileToken, + } +} + +// A drive docx goes through the blocks path: main Markdown item (external_id = +// file token, channel = feishu_drive) plus the attachment sub-item. +func TestDriveFetchStream_DocxBlocksMultiItem(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") + const ( + attToken = "ft-drv-att" + attName = "report.pdf" + ) + attContent := bytes.Repeat([]byte("x"), core.MinAttachmentBytes+1) + + _, cfg := fakeFeishuDriveDocx(t, []core.DriveFile{driveDocxFile()}, driveDocxFileToken, + driveDocxBlocks(attToken, attName), "ok", attContent) + + c := NewDriveConnector(core.RegionFeishuDrive) + h := &recordingHandler{} + _, err := c.FetchStream(context.Background(), makeDriveConfig(cfg, []string{"folder1"}, false), nil, h) + if err != nil { + t.Fatalf("FetchStream() error: %v", err) + } + + if len(h.emitted) != 2 { + t.Fatalf("expected 2 emitted items (main doc + attachment), got %d: %+v", len(h.emitted), h.emitted) + } + + main := h.emitted[0] + if main.ExternalID != driveDocxFileToken { + t.Errorf("items[0].ExternalID = %q, want %q", main.ExternalID, driveDocxFileToken) + } + if main.ContentType != "text/markdown" { + t.Errorf("items[0].ContentType = %q, want text/markdown", main.ContentType) + } + if !main.ReplacesSubtree { + t.Errorf("items[0].ReplacesSubtree = false, want true") + } + if main.Metadata["channel"] != types.ChannelFeishuDrive { + t.Errorf("items[0].Metadata[channel] = %q, want %q", main.Metadata["channel"], types.ChannelFeishuDrive) + } + if main.URL != "https://example.feishu.cn/file/"+driveDocxFileToken { + t.Errorf("items[0].URL = %q, want drive file URL passthrough", main.URL) + } + if !strings.Contains(string(main.Content), "Hello drive") { + t.Errorf("items[0].Content missing expected text; got %q", string(main.Content)) + } + + att := h.emitted[1] + wantAttID := driveDocxFileToken + "#file#" + attToken + if att.ExternalID != wantAttID { + t.Errorf("items[1].ExternalID = %q, want %q", att.ExternalID, wantAttID) + } + if att.Metadata["attachment"] != "true" { + t.Errorf("items[1].Metadata[attachment] = %q, want \"true\"", att.Metadata["attachment"]) + } +} + +// Blocks API failure falls back to export: exactly one octet-stream item, no +// ReplacesSubtree (must not sweep good prior children on a transient failure). +func TestDriveFetchStream_DocxBlocksFailFallsBackToExport(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") + _, cfg := fakeFeishuDriveDocx(t, []core.DriveFile{driveDocxFile()}, driveDocxFileToken, nil, "fail", nil) + + c := NewDriveConnector(core.RegionFeishuDrive) + h := &recordingHandler{} + _, err := c.FetchStream(context.Background(), makeDriveConfig(cfg, []string{"folder1"}, false), nil, h) + if err != nil { + t.Fatalf("FetchStream() error: %v", err) + } + + if len(h.emitted) != 1 { + t.Fatalf("expected 1 emitted item (export fallback), got %d: %+v", len(h.emitted), h.emitted) + } + item := h.emitted[0] + if item.ExternalID != driveDocxFileToken { + t.Errorf("item.ExternalID = %q, want %q", item.ExternalID, driveDocxFileToken) + } + if item.ContentType != "application/octet-stream" { + t.Errorf("item.ContentType = %q, want application/octet-stream", item.ContentType) + } + if !strings.HasSuffix(item.FileName, ".docx") { + t.Errorf("item.FileName = %q, want .docx suffix", item.FileName) + } + if item.ReplacesSubtree { + t.Error("export-fallback item must not set ReplacesSubtree") + } +} + +// Blocks rendering to empty Markdown also falls back to export (a blank page +// would otherwise ingest as a login-gated URL core.Fetch and fail). +func TestDriveFetchStream_DocxBlocksEmptyFallsBackToExport(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") + _, cfg := fakeFeishuDriveDocx(t, []core.DriveFile{driveDocxFile()}, driveDocxFileToken, nil, "empty", nil) + + c := NewDriveConnector(core.RegionFeishuDrive) + h := &recordingHandler{} + _, err := c.FetchStream(context.Background(), makeDriveConfig(cfg, []string{"folder1"}, false), nil, h) + if err != nil { + t.Fatalf("FetchStream() error: %v", err) + } + + if len(h.emitted) != 1 { + t.Fatalf("expected 1 emitted item (export fallback), got %d: %+v", len(h.emitted), h.emitted) + } + item := h.emitted[0] + if item.ContentType != "application/octet-stream" { + t.Errorf("item.ContentType = %q, want application/octet-stream", item.ContentType) + } + if item.ReplacesSubtree { + t.Error("export-fallback item must not set ReplacesSubtree") + } +} + +// With multimodal disabled, an embedded image is not downloaded or emitted, +// but its external_id is still in SubtreeKeep so a later toggle-on does not +// sweep it, and toggling VLM off later does not delete previously OCR'd images. +func TestDriveFetchStream_DocxImageMultimodalOff(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") + const imgToken = "img-drv-1" + blocks := []core.DocxBlock{ + {BlockID: "b1", BlockType: core.BlockTypePage}, + {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ + Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "has image"}}}, + }}, + {BlockID: "b3", BlockType: core.BlockTypeImage, Image: &core.BlockTokenRef{Token: imgToken}}, + } + _, cfg := fakeFeishuDriveDocx(t, []core.DriveFile{driveDocxFile()}, driveDocxFileToken, blocks, "ok", nil) + + c := NewDriveConnector(core.RegionFeishuDrive) + h := &recordingHandler{} + _, err := c.FetchStream(context.Background(), makeDriveConfig(cfg, []string{"folder1"}, false), nil, h) + if err != nil { + t.Fatalf("FetchStream() error: %v", err) + } + + if len(h.emitted) != 1 { + t.Fatalf("expected 1 emitted item (main only, image core.Skipped), got %d: %+v", len(h.emitted), h.emitted) + } + main := h.emitted[0] + wantKeep := driveDocxFileToken + "#image#" + imgToken + found := false + for _, k := range main.SubtreeKeep { + if k == wantKeep { + found = true + } + } + if !found { + t.Errorf("SubtreeKeep = %v, want it to contain %q", main.SubtreeKeep, wantKeep) + } +} diff --git a/internal/datasource/connector/feishu/drive/helpers_test.go b/internal/datasource/connector/feishu/drive/helpers_test.go new file mode 100644 index 000000000..59056b1cc --- /dev/null +++ b/internal/datasource/connector/feishu/drive/helpers_test.go @@ -0,0 +1,61 @@ +package drive + +import ( + "context" + "encoding/json" + "net/http" + "os" + "testing" + + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" + "github.com/Tencent/WeKnora/internal/types" + secutils "github.com/Tencent/WeKnora/internal/utils" +) + +func TestMain(m *testing.M) { + os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + secutils.ResetSSRFWhitelistForTest() + os.Exit(m.Run()) +} + +func writeJSON(w http.ResponseWriter, v interface{}) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +func makeConfig(cfg *core.Config, resourceIDs []string) *types.DataSourceConfig { + creds := map[string]interface{}{ + "app_id": cfg.AppID, + "app_secret": cfg.AppSecret, + "base_url": cfg.BaseURL, + } + return &types.DataSourceConfig{ + Type: types.ConnectorTypeFeishu, + Credentials: creds, + ResourceIDs: resourceIDs, + } +} + +type recordingHandler struct { + emitted []types.FetchedItem + checkpoints []core.FeishuDriveCursor + emitErr func(item types.FetchedItem) error +} + +func (h *recordingHandler) Emit(_ context.Context, item types.FetchedItem) error { + if h.emitErr != nil { + if err := h.emitErr(item); err != nil { + return err + } + } + h.emitted = append(h.emitted, item) + return nil +} + +func (h *recordingHandler) Checkpoint(_ context.Context, cursor *types.SyncCursor) error { + var fc core.FeishuDriveCursor + b, _ := json.Marshal(cursor.ConnectorCursor) + _ = json.Unmarshal(b, &fc) + h.checkpoints = append(h.checkpoints, fc) + return nil +} diff --git a/internal/datasource/connector/feishu/types.go b/internal/datasource/connector/feishu/types.go deleted file mode 100644 index f57823074..000000000 --- a/internal/datasource/connector/feishu/types.go +++ /dev/null @@ -1,230 +0,0 @@ -// Package feishu implements the Feishu (飞书/Lark) data source connector for WeKnora. -// -// It syncs documents from Feishu Wiki spaces and cloud documents into WeKnora knowledge bases. -// -// Feishu API docs: -// - Wiki spaces: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space/list -// - Wiki nodes: https://open.feishu.cn/document/server-docs/docs/wiki-v2/space-node/list -// - Export tasks: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/export-user-guide -// - File download: https://open.feishu.cn/document/server-docs/docs/drive-v1/file/download -// - Auth: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal -package feishu - -import "time" - -// Config holds Feishu-specific configuration for the data source connector. -// Uses the self-built app (企业自建应用) authentication model. -type Config struct { - // App ID from Feishu developer console - AppID string `json:"app_id"` - - // App Secret from Feishu developer console - AppSecret string `json:"app_secret"` - - // Base URL for Feishu API (default: https://open.feishu.cn) - // Use https://open.larksuite.com for Lark (international) deployments - BaseURL string `json:"base_url,omitempty"` - - // Timezone is the IANA name (e.g. "Asia/Shanghai", "America/New_York") used to - // render bitable date cells. Feishu stores a date as a UTC instant, but the - // calendar date a user sees is that instant in the table's timezone, so - // rendering in UTC shifts the date. Empty defaults to GMT+8, which matches - // Feishu (mainland) tenants; set it for Lark tenants in other zones. - Timezone string `json:"timezone,omitempty"` -} - -// defaultTimezoneOffsetSeconds is GMT+8 (the Feishu mainland default), used when -// no timezone is configured. A fixed zone avoids any dependency on system tzdata, -// which minimal container images may omit. -const defaultTimezoneOffsetSeconds = 8 * 3600 - -// resolveLocation returns the *time.Location used to render bitable date cells. -// An empty name yields a fixed GMT+8 zone (no tzdata dependency); a named zone is -// loaded from the system tz database, falling back to GMT+8 if it is unavailable. -func resolveLocation(name string) *time.Location { - if name != "" { - if loc, err := time.LoadLocation(name); err == nil { - return loc - } - } - return time.FixedZone("GMT+8", defaultTimezoneOffsetSeconds) -} - -// DefaultBaseURL is the default Feishu Open Platform API base URL. -const DefaultBaseURL = "https://open.feishu.cn" - -// LarkBaseURL is the Lark (international) API base URL. -const LarkBaseURL = "https://open.larksuite.com" - -// GetBaseURL returns the effective base URL, defaulting to Feishu if not set. -func (c *Config) GetBaseURL() string { - if c.BaseURL != "" { - return c.BaseURL - } - return DefaultBaseURL -} - -// --- Export format constants --- -// Used by the export task API: POST /drive/v1/export_tasks - -const ( - // ExportTypeDocx exports Feishu documents to .docx format. - ExportTypeDocx = "docx" - // ExportTypeXlsx exports spreadsheets / bitable to .xlsx format. - ExportTypeXlsx = "xlsx" - // ExportTypePDF exports documents to .pdf format (fallback). - ExportTypePDF = "pdf" -) - -// objTypeToExportFileExtension maps Feishu obj_type to the best export file_extension. -var objTypeToExportFileExtension = map[string]string{ - "docx": ExportTypeDocx, - "doc": ExportTypeDocx, - "sheet": ExportTypeXlsx, - "bitable": ExportTypeXlsx, -} - -// objTypeToExportType maps Feishu obj_type to the export API "type" parameter. -// See: https://open.feishu.cn/document/server-docs/docs/drive-v1/export_task/create -var objTypeToExportType = map[string]string{ - "docx": "docx", - "doc": "doc", - "sheet": "sheet", - "bitable": "bitable", -} - -// exportFileExtToSuffix maps export file_extension to the file suffix for FileName. -var exportFileExtToSuffix = map[string]string{ - ExportTypeDocx: ".docx", - ExportTypeXlsx: ".xlsx", - ExportTypePDF: ".pdf", -} - -// --- Feishu API response structures --- - -// apiResponse is the common Feishu API response wrapper. -type apiResponse struct { - Code int `json:"code"` - Msg string `json:"msg"` -} - -// tokenResponse is the response for tenant_access_token API. -type tokenResponse struct { - apiResponse - TenantAccessToken string `json:"tenant_access_token"` - Expire int `json:"expire"` // seconds -} - -// wikiSpaceListResponse is the response for GET /open-apis/wiki/v2/spaces. -type wikiSpaceListResponse struct { - apiResponse - Data struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - } `json:"data"` -} - -// wikiSpace represents a Feishu Wiki space. -type wikiSpace struct { - SpaceID string `json:"space_id"` - Name string `json:"name"` - Description string `json:"description"` - Visibility string `json:"visibility"` // "public" or "private" -} - -// wikiNodeListResponse is the response for GET /open-apis/wiki/v2/spaces/:space_id/nodes. -type wikiNodeListResponse struct { - apiResponse - Data struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - } `json:"data"` -} - -// wikiNode represents a node (document or folder) in a Feishu Wiki space. -type wikiNode struct { - SpaceID string `json:"space_id"` - NodeToken string `json:"node_token"` - ObjToken string `json:"obj_token"` // document token - ObjType string `json:"obj_type"` // "doc", "sheet", "mindnote", "bitable", "file", "docx", "slides" - ParentNodeID string `json:"parent_node_token"` - NodeType string `json:"node_type"` // "origin" or "shortcut" - OriginNodeID string `json:"origin_node_id"` - OriginSpaceID string `json:"origin_space_id"` - HasChild bool `json:"has_child"` - Title string `json:"title"` - Creator string `json:"creator"` - Owner string `json:"owner"` - ObjCreateTime string `json:"obj_create_time"` // document creation time (unix timestamp string) - ObjEditTime string `json:"obj_edit_time"` // document last edit time (unix timestamp string) — tracks content changes - NodeCreateTime string `json:"node_create_time"` // node creation time (unix timestamp string) - NodeEditTime string `json:"node_edit_time"` // node edit time (unix timestamp string) — only tracks node attribute changes -} - -// wikiNodeInfoResponse is the response for GET /open-apis/wiki/v2/spaces/get_node. -type wikiNodeInfoResponse struct { - apiResponse - Data struct { - Node wikiNode `json:"node"` - } `json:"data"` -} - -// --- Export task API responses --- - -// docRawContentResponse is the response for GET /open-apis/docx/v1/documents/:document_id/raw_content. -// Deprecated: prefer export API for full-fidelity document export. -type docRawContentResponse struct { - apiResponse - Data struct { - Content string `json:"content"` - } `json:"data"` -} - -// exportTaskCreateResponse is the response for POST /drive/v1/export_tasks. -type exportTaskCreateResponse struct { - apiResponse - Data struct { - Ticket string `json:"ticket"` - } `json:"data"` -} - -// exportTaskStatusResponse is the response for GET /drive/v1/export_tasks/{ticket}. -type exportTaskStatusResponse struct { - apiResponse - Data struct { - Result struct { - FileToken string `json:"file_token"` - FileSize int64 `json:"file_size"` - // JobStatus: 0=success, 1=initializing, 2=processing - JobStatus int `json:"job_status"` - JobErrorMsg string `json:"job_error_msg"` - FileName string `json:"file_name"` - } `json:"result"` - } `json:"data"` -} - -// --- File download response --- - -// driveFileMetaResponse is the response for GET /drive/v1/metas for file type nodes. -type driveFileMetaResponse struct { - apiResponse - Data struct { - Metas []struct { - DocToken string `json:"doc_token"` - DocType string `json:"doc_type"` - Title string `json:"title"` - } `json:"metas"` - } `json:"data"` -} - -// feishuCursor stores incremental sync state for Feishu. -type feishuCursor struct { - // LastSyncTime is the timestamp of the last successful sync. - LastSyncTime time.Time `json:"last_sync_time"` - - // SpaceNodeTimes maps space_id -> node_token -> last known edit time. - // Used to detect which nodes have changed since last sync. - SpaceNodeTimes map[string]map[string]string `json:"space_node_times,omitempty"` -} diff --git a/internal/datasource/connector/feishu/wiki/connector.go b/internal/datasource/connector/feishu/wiki/connector.go new file mode 100644 index 000000000..1fc359444 --- /dev/null +++ b/internal/datasource/connector/feishu/wiki/connector.go @@ -0,0 +1,467 @@ +package wiki + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/Tencent/WeKnora/internal/datasource" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" + "github.com/Tencent/WeKnora/internal/logger" + "github.com/Tencent/WeKnora/internal/types" +) + +// Connector implements the datasource.Connector interface for Feishu and, with +// the same code, for Lark: the two clouds expose an identical wiki/docx/drive +// API surface. A core.Region picks the cloud — see region.go. +type Connector struct { + region core.Region +} + +// NewConnector creates a connector for the given region (RegionFeishu or RegionLark). +func NewConnector(region core.Region) *Connector { + return &Connector{region: region} +} + +// Feishu supports resumable streaming sync; the service prefers FetchStream over +// FetchAll/FetchIncremental when a connector implements StreamingConnector. +var _ datasource.StreamingConnector = (*Connector)(nil) + +// Type returns the connector type identifier. +func (c *Connector) Type() string { + return c.region.ConnectorType +} + +// Validate verifies that the Feishu configuration is valid by testing connectivity. +func (c *Connector) Validate(ctx context.Context, config *types.DataSourceConfig) error { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return err + } + + client := core.NewClient(feishuConfig) + if err := client.Ping(ctx); err != nil { + return fmt.Errorf("feishu connection failed: %w", err) + } + + return nil +} + +// ListResources lists Feishu Wiki resources for selection, loading the tree +// lazily one level at a time to avoid traversing the entire wiki up front. +// +// - parentID == "" → list all accessible wiki spaces. +// - parentID == spaceID → list the top-level nodes of that space. +// - parentID == "spaceID:nodeToken" → list the direct children of that node. +// +// Eagerly recursing the whole tree here used to time out for large wikis +// (Tencent/WeKnora#1672); the recursive walk now happens only at sync time. +func (c *Connector) ListResources( + ctx context.Context, config *types.DataSourceConfig, parentID string, +) ([]types.Resource, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + + client := core.NewClient(feishuConfig) + + if parentID == "" { + spaces, err := client.ListWikiSpaces(ctx) + if err != nil { + return nil, fmt.Errorf("list feishu wiki spaces: %w", err) + } + + resources := make([]types.Resource, 0, len(spaces)) + for _, space := range spaces { + resources = append(resources, types.Resource{ + ExternalID: space.SpaceID, + Name: space.Name, + Type: "wiki_space", + Description: space.Description, + URL: c.region.WikiURL(space.SpaceID), + HasChildren: true, + Metadata: map[string]interface{}{ + "visibility": space.Visibility, + "space_id": space.SpaceID, + }, + }) + } + return resources, nil + } + + // Lazy load: list only the direct children of the given space / node. + spaceID, nodeToken := parseWikiResourceID(parentID) + nodes, err := client.ListWikiNodes(ctx, spaceID, nodeToken) + if err != nil { + return nil, fmt.Errorf("list feishu wiki nodes under %s: %w", parentID, err) + } + + resources := make([]types.Resource, 0, len(nodes)) + for _, node := range nodes { + resources = append(resources, c.wikiNodeToResource(spaceID, node)) + } + return resources, nil +} + +// ResolveResourceAncestors returns the resource IDs of every parent that has to +// be expanded so the lazily-loaded picker can reveal each given selection. For a +// selected node "spaceID:nodeToken" that is its space plus every intermediate +// node up the tree; the walk uses GetWikiNode (parent_node_token) and is O(depth) +// per selection, so it never re-traverses the whole wiki. +func (c *Connector) ResolveResourceAncestors( + ctx context.Context, config *types.DataSourceConfig, resourceIDs []string, +) ([]string, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + client := core.NewClient(feishuConfig) + + seen := make(map[string]bool) + ancestors := make([]string, 0) + add := func(id string) { + if id != "" && !seen[id] { + seen[id] = true + ancestors = append(ancestors, id) + } + } + + for _, rid := range resourceIDs { + spaceID, nodeToken := parseWikiResourceID(rid) + if spaceID == "" || nodeToken == "" { + // A space-level selection is already a top-level node in the picker; + // there is nothing above it to reveal. + continue + } + // The space's direct children must be loaded to reveal the top-level node. + add(spaceID) + + // Walk up from the selection to the top, loading each intermediate + // parent so the path down to the selection becomes visible. + current := nodeToken + for current != "" { + node, err := client.GetWikiNode(ctx, spaceID, current) + if err != nil { + // Best-effort: a broken path just stays collapsed, the rest of + // the selections are still revealed. + logger.Warnf(ctx, "[Feishu] resolve ancestors: get node %s:%s: %v", spaceID, current, err) + break + } + if node.ParentNodeID == "" { + break + } + add(makeWikiNodeResourceID(spaceID, node.ParentNodeID)) + current = node.ParentNodeID + } + } + + return ancestors, nil +} + +// FetchAll performs a full sync of all documents from the specified wiki spaces. +// Defensive fallback path - the service prefers FetchStream when the connector +// implements StreamingConnector. +func (c *Connector) FetchAll(ctx context.Context, config *types.DataSourceConfig, resourceIDs []string) ([]types.FetchedItem, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + client := core.NewClient(feishuConfig) + return core.FetchAllEngine(ctx, client, config, resourceIDs, wikiOps{region: c.region}) +} + +// FetchIncremental performs an incremental sync by comparing node edit times +// against the previously recorded state. Defensive fallback path - the service +// prefers FetchStream. Routed through the same engine, so the #2136 +// failure-doesn't-advance-cursor semantics apply here too (previously this path +// advanced the cursor before fetching, a latent #2136 bug). +func (c *Connector) FetchIncremental(ctx context.Context, config *types.DataSourceConfig, cursor *types.SyncCursor) ([]types.FetchedItem, *types.SyncCursor, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, nil, err + } + client := core.NewClient(feishuConfig) + ops := wikiOps{region: c.region} + if len(config.ResourceIDs) == 0 { + return nil, nil, errors.New(ops.EmptyResourceIDsError()) + } + return core.FetchIncrementalEngine(ctx, client, config, cursor, ops) +} + +// FetchStream performs a resumable, memory-bounded sync. It unifies the full +// and incremental paths: with cursor == nil it fetches everything, and with a +// cursor it skips nodes whose recorded edit time is unchanged - the same +// mechanism that lets a sync which timed out mid-traversal resume from the last +// checkpoint instead of restarting (Tencent/WeKnora#2136). +// +// The per-node loop lives in the shared engine (engine.go); this shell only +// wires the wiki NodeOps adapter. +func (c *Connector) FetchStream( + ctx context.Context, config *types.DataSourceConfig, + cursor *types.SyncCursor, h datasource.StreamHandler, +) (*types.SyncCursor, error) { + feishuConfig, err := core.ParseFeishuConfig(config, c.region) + if err != nil { + return nil, err + } + client := core.NewClient(feishuConfig) + ops := wikiOps{region: c.region} + if len(config.ResourceIDs) == 0 { + return nil, errors.New(ops.EmptyResourceIDsError()) + } + return core.FetchStreamEngine(ctx, client, config, cursor, h, ops) +} + +// wikiOps adapts the wiki Connector to the generic sync engine. It carries the +// region (for URL rendering) and encodes/decodes the wiki cursor wire format +// (core.FeishuCursor / space_node_times) so the engine can stay format-agnostic. +type wikiOps struct { + region core.Region +} + +func (o wikiOps) List(ctx context.Context, client *core.Client, resourceID string) ([]core.WikiNode, error, error) { + spaceID, nodeToken := parseWikiResourceID(resourceID) + nodes, err := client.ListWikiNodesRecursiveFrom(ctx, spaceID, nodeToken) + if err == nil { + return nodes, nil, nil + } + var partial *core.PartialWikiNodeListError + if errors.As(err, &partial) { + // Partial listing: nodes are still usable; the failed sub-trees are + // surfaced via ListFailureItems, and the sync continues. + return nodes, err, nil + } + return nodes, nil, err +} + +func (o wikiOps) Token(n core.WikiNode) string { return n.NodeToken } +func (o wikiOps) Title(n core.WikiNode) string { return n.Title } +func (o wikiOps) ObjType(n core.WikiNode) string { return n.ObjType } + +// EditTime is the change-detection timestamp: ObjEditTime (document content) +// with a NodeEditTime fallback for nodes that lack obj_edit_time. This drives +// the cursor comparison, NOT FetchedItem.UpdatedAt (which uses NodeEditTime). +func (o wikiOps) EditTime(n core.WikiNode) string { + if n.ObjEditTime != "" { + return n.ObjEditTime + } + return n.NodeEditTime +} + +func (o wikiOps) Fetch(ctx context.Context, client *core.Client, n core.WikiNode, resourceID string, multimodal bool) ([]*types.FetchedItem, error) { + spaceID, _ := parseWikiResourceID(resourceID) + return fetchNodeContent(ctx, client, n, spaceID, resourceID, multimodal, o.region) +} + +func (o wikiOps) ListFailureItems(resourceID string, partial error) []types.FetchedItem { + spaceID, _ := parseWikiResourceID(resourceID) + var pe *core.PartialWikiNodeListError + if errors.As(partial, &pe) { + return appendWikiNodeListFailureItems(nil, spaceID, resourceID, pe.Failures) + } + return nil +} + +func (o wikiOps) ResourceNoun() string { return "nodes" } +func (o wikiOps) EmptyResourceIDsError() string { + return "no resource IDs (wiki space IDs or wiki node IDs) configured" +} +func (o wikiOps) LogTag() string { return "[Feishu]" } + +func (o wikiOps) DecodeCursorTimes(m map[string]interface{}) map[string]map[string]string { + var prev core.FeishuCursor + b, _ := json.Marshal(m) + _ = json.Unmarshal(b, &prev) + return prev.SpaceNodeTimes +} + +func (o wikiOps) EncodeCursor(times map[string]map[string]string, lastSync time.Time) *types.SyncCursor { + fc := core.FeishuCursor{LastSyncTime: lastSync, SpaceNodeTimes: times} + m := make(map[string]interface{}) + b, _ := json.Marshal(fc) + _ = json.Unmarshal(b, &m) + return &types.SyncCursor{LastSyncTime: lastSync, ConnectorCursor: m} +} + +func appendWikiNodeListFailureItems(items []types.FetchedItem, spaceID string, resourceID string, failures []core.WikiNodeListFailure) []types.FetchedItem { + for _, failure := range failures { + node := failure.Node + title := node.Title + if title == "" { + title = node.NodeToken + } + items = append(items, types.FetchedItem{ + ExternalID: node.NodeToken, + Title: title, + SourceResourceID: resourceID, + Metadata: core.FeishuErrorItemMeta(failure.Err, map[string]string{ + "channel": types.ChannelFeishu, + "node_token": node.NodeToken, + "space_id": spaceID, + "failure_stage": "list_children", + }), + }) + } + return items +} + +// fetchNodeContent fetches the content of a single wiki node and converts it to a +// slice of FetchedItems. For docx nodes it fans out into a main Markdown document +// plus optional attachment sub-items. Dispatches to different retrieval strategies +// based on obj_type: +// - docx → blocks API (Markdown) with export fallback; may return attachments +// - doc/sheet/bitable → export API → binary file +// - file → drive download → original file (PDF/Word/image/etc.) +// - mindnote → Skip (no API) +// - slides → Skip (no API) +func fetchNodeContent(ctx context.Context, client *core.Client, node core.WikiNode, spaceID string, resourceID string, multimodalEnabled bool, region core.Region) ([]*types.FetchedItem, error) { + if !core.IsSupportedDocType(node.ObjType) { + return nil, nil + } + + editTime := core.ParseFeishuTimestamp(node.NodeEditTime) + baseMeta := map[string]string{ + "obj_token": node.ObjToken, + "obj_type": node.ObjType, + "node_token": node.NodeToken, + "space_id": spaceID, + "creator": node.Creator, + "owner": node.Owner, + "channel": types.ChannelFeishu, + } + + switch node.ObjType { + case "docx": + return core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ + DocToken: node.NodeToken, + ObjToken: node.ObjToken, + Title: node.Title, + URL: region.WikiURL(node.NodeToken), + ResourceID: resourceID, + EditTime: editTime, + BaseMeta: baseMeta, + MultimodalEnabled: multimodalEnabled, + }) + case "doc", "sheet", "bitable": + item, err := fetchViaExport(ctx, client, node, resourceID, editTime, baseMeta, region) + if err != nil { + return nil, err + } + return []*types.FetchedItem{item}, nil + case "file": + item, err := fetchDriveFile(ctx, client, node, resourceID, editTime, baseMeta, region) + if err != nil { + return nil, err + } + return []*types.FetchedItem{item}, nil + default: + return nil, nil + } +} + +// fetchViaExport exports a doc/sheet/bitable node via the async export API and +// returns a single FetchedItem containing the exported binary. +func fetchViaExport(ctx context.Context, client *core.Client, node core.WikiNode, resourceID string, editTime time.Time, baseMeta map[string]string, region core.Region) (*types.FetchedItem, error) { + // Export as a file via the async export API + data, fileName, err := client.ExportAndDownload(ctx, node.ObjToken, node.ObjType) + if err != nil { + return nil, fmt.Errorf("export %s (%s): %w", node.Title, node.ObjType, err) + } + + // Ensure a reasonable file name with correct extension + ext := core.ExportFileExtToSuffix[core.ObjTypeToExportFileExtension[node.ObjType]] + if fileName == "" { + fileName = core.SanitizeFileName(node.Title) + ext + } else if !strings.HasSuffix(strings.ToLower(fileName), ext) { + // Feishu often returns the doc title without extension - append it + fileName = core.SanitizeFileName(fileName) + ext + } + + return &types.FetchedItem{ + ExternalID: node.NodeToken, + Title: node.Title, + Content: data, + ContentType: "application/octet-stream", + FileName: fileName, + URL: region.WikiURL(node.NodeToken), + UpdatedAt: editTime, + SourceResourceID: resourceID, + Metadata: baseMeta, + }, nil +} + +// fetchDriveFile downloads an original uploaded file from Drive and returns a +// single FetchedItem containing the raw bytes. +func fetchDriveFile(ctx context.Context, client *core.Client, node core.WikiNode, resourceID string, editTime time.Time, baseMeta map[string]string, region core.Region) (*types.FetchedItem, error) { + // Download the original uploaded file from Drive + data, err := client.DownloadDriveFile(ctx, node.ObjToken) + if err != nil { + return nil, fmt.Errorf("download file %s (%s): %w", node.Title, node.ObjToken, err) + } + + // Use the node title as file name; it usually preserves the original extension + fileName := node.Title + if fileName == "" { + fileName = node.ObjToken + } + + return &types.FetchedItem{ + ExternalID: node.NodeToken, + Title: node.Title, + Content: data, + ContentType: "application/octet-stream", + FileName: fileName, + URL: region.WikiURL(node.NodeToken), + UpdatedAt: editTime, + SourceResourceID: resourceID, + Metadata: baseMeta, + }, nil +} + +// --- Helper functions --- + +func makeWikiNodeResourceID(spaceID, nodeToken string) string { + return spaceID + core.FeishuWikiNodeResourceSeparator + nodeToken +} + +func parseWikiResourceID(resourceID string) (spaceID string, nodeToken string) { + spaceID, nodeToken, _ = strings.Cut(resourceID, core.FeishuWikiNodeResourceSeparator) + return spaceID, nodeToken +} + +func (c *Connector) wikiNodeToResource(spaceID string, node core.WikiNode) types.Resource { + parentID := spaceID + if node.ParentNodeID != "" { + parentID = makeWikiNodeResourceID(spaceID, node.ParentNodeID) + } + + name := node.Title + if name == "" { + name = node.NodeToken + } + + modifiedAt := core.ParseFeishuTimestamp(node.ObjEditTime) + if modifiedAt.IsZero() { + modifiedAt = core.ParseFeishuTimestamp(node.NodeEditTime) + } + + return types.Resource{ + ExternalID: makeWikiNodeResourceID(spaceID, node.NodeToken), + Name: name, + Type: "wiki_node", + URL: c.region.WikiURL(node.NodeToken), + ParentID: parentID, + HasChildren: node.HasChild, + ModifiedAt: modifiedAt, + Metadata: map[string]interface{}{ + "space_id": spaceID, + "node_token": node.NodeToken, + "obj_token": node.ObjToken, + "obj_type": node.ObjType, + }, + } +} diff --git a/internal/datasource/connector/feishu/connector_convergence_test.go b/internal/datasource/connector/feishu/wiki/connector_convergence_test.go similarity index 81% rename from internal/datasource/connector/feishu/connector_convergence_test.go rename to internal/datasource/connector/feishu/wiki/connector_convergence_test.go index c1c17013d..16961b387 100644 --- a/internal/datasource/connector/feishu/connector_convergence_test.go +++ b/internal/datasource/connector/feishu/wiki/connector_convergence_test.go @@ -1,4 +1,4 @@ -package feishu +package wiki import ( "context" @@ -9,6 +9,7 @@ import ( "sync" "testing" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/types" ) @@ -36,31 +37,23 @@ func (s *statefulFeishu) shouldFail(token string) bool { return s.failTokens[token] } -func newStatefulFeishu(nodes []wikiNode) (*httptest.Server, *Config, *statefulFeishu) { +func newStatefulFeishu(nodes []core.WikiNode) (*httptest.Server, *core.Config, *statefulFeishu) { s := &statefulFeishu{failTokens: map[string]bool{}} mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) + writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, + writeJSON(w, core.WikiSpaceListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiNodeListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: nodes}, + writeJSON(w, core.WikiNodeListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeListData{Items: nodes}, }) }) // Export create: ticket == obj_token so the poll below can key on it. @@ -69,9 +62,9 @@ func newStatefulFeishu(nodes []wikiNode) (*httptest.Server, *Config, *statefulFe Token string `json:"token"` } _ = json.NewDecoder(r.Body).Decode(&body) - writeJSON(w, exportTaskCreateResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct{ Ticket string `json:"ticket"` }{Ticket: body.Token}, + writeJSON(w, core.ExportTaskCreateResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskCreateData{Ticket: body.Token}, }) }) // Export status poll: /open-apis/drive/v1/export_tasks/?token= @@ -86,7 +79,7 @@ func newStatefulFeishu(nodes []wikiNode) (*httptest.Server, *Config, *statefulFe if s.shouldFail(token) { status = 3 // failed job } - resp := exportTaskStatusResponse{apiResponse: apiResponse{Code: 0}} + resp := core.ExportTaskStatusResponse{ApiResponse: core.ApiResponse{Code: 0}} resp.Data.Result.FileToken = "file-" + token resp.Data.Result.FileName = "exported.docx" resp.Data.Result.JobStatus = status @@ -95,7 +88,7 @@ func newStatefulFeishu(nodes []wikiNode) (*httptest.Server, *Config, *statefulFe }) ts := httptest.NewServer(mux) - return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}, s + return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL}, s } // convergenceHandler models the service side across a resumable sync: @@ -107,7 +100,7 @@ type convergenceHandler struct { ingested []string // ExternalIDs of successfully ingested content items failed []string // ExternalIDs surfaced as failures (Metadata["error"]) exportedTokens map[string]int - checkpoints []feishuCursor + checkpoints []core.FeishuCursor calls int cancelAfterCall int // >0: cancel ctx and abort on the Nth Emit call cancel context.CancelFunc @@ -133,7 +126,7 @@ func (h *convergenceHandler) Emit(ctx context.Context, item types.FetchedItem) e } func (h *convergenceHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error { - var fc feishuCursor + var fc core.FeishuCursor b, _ := json.Marshal(cursor.ConnectorCursor) _ = json.Unmarshal(b, &fc) h.checkpoints = append(h.checkpoints, fc) @@ -157,17 +150,17 @@ func lastCheckpointCursor(h *convergenceHandler, t *testing.T) *types.SyncCursor // TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure is the // end-to-end proof for Tencent/WeKnora#2136: a large-ish wiki that (a) hits a // transient per-node export failure and (b) is killed by the 2h task timeout -// mid-traversal must, on the asynq retry, resume from the last checkpoint, -// re-fetch only what is outstanding, retry the transiently-failed node, and end -// with EVERY document synced exactly once — no permanent skip, no full restart, +// mid-traversal must, on the asynq retry, resume from the last Checkpoint, +// re-core.Fetch only what is outstanding, retry the transiently-failed node, and end +// with EVERY document synced exactly once — no permanent core.Skip, no full restart, // no redundant re-export of already-done nodes. func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T) { // Checkpoint on every processed node so the persisted cursor is precise. - prevN := feishuStreamCheckpointInterval - feishuStreamCheckpointInterval = 1 - defer func() { feishuStreamCheckpointInterval = prevN }() + prevN := core.FeishuStreamCheckpointInterval + core.FeishuStreamCheckpointInterval = 1 + defer func() { core.FeishuStreamCheckpointInterval = prevN }() - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", ObjEditTime: "100"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", ObjEditTime: "200"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"}, @@ -177,7 +170,7 @@ func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T ts, cfg, srv := newStatefulFeishu(nodes) defer ts.Close() cfgDS := makeConfig(cfg, []string{"space1"}) - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) // ---- Pass 1: obj2 export fails transiently; task "times out" on the 3rd // Emit call (nt1 success, nt2 failure-item, then cancel as nt3 is emitted). @@ -195,14 +188,14 @@ func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T t.Fatalf("pass 1 ingested = %q, want just nt1", got) } - // The persisted cursor (last checkpoint) must contain nt1 but NOT the - // aborted nt3, nor the transiently-failed nt2 — otherwise resume would skip + // The persisted cursor (last Checkpoint) must contain nt1 but NOT the + // aborted nt3, nor the transiently-failed nt2 — otherwise resume would core.Skip // them forever. persisted := lastCheckpointCursor(h1, t) if persisted == nil { - t.Fatalf("pass 1 wrote no checkpoint — resume would restart from scratch") + t.Fatalf("pass 1 wrote no Checkpoint — resume would restart from scratch") } - var pc feishuCursor + var pc core.FeishuCursor pb, _ := json.Marshal(persisted.ConnectorCursor) _ = json.Unmarshal(pb, &pc) p := pc.SpaceNodeTimes["space1"] @@ -210,10 +203,10 @@ func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T t.Errorf("persisted cursor missing nt1 (done work lost on resume)") } if _, ok := p["nt2"]; ok { - t.Errorf("persisted cursor recorded transiently-failed nt2 — it would be skipped forever") + t.Errorf("persisted cursor recorded transiently-failed nt2 — it would be core.Skipped forever") } if _, ok := p["nt3"]; ok { - t.Errorf("persisted cursor recorded aborted nt3 — it would be skipped forever") + t.Errorf("persisted cursor recorded aborted nt3 — it would be core.Skipped forever") } // ---- Pass 2: asynq retry. obj2 has recovered; run to completion resuming @@ -225,7 +218,7 @@ func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T t.Fatalf("pass 2 error: %v", err2) } - // nt1 was already done → must be skipped (not re-exported, not re-ingested). + // nt1 was already done → must be core.Skipped (not re-exported, not re-ingested). for _, id := range h2.ingested { if id == "nt1" { t.Errorf("pass 2 re-ingested nt1 — redundant re-export of completed work") @@ -263,12 +256,12 @@ func TestFetchStream_ResumeConvergesAfterTimeoutAndTransientFailure(t *testing.T // Final cursor is a complete snapshot of every node (next incremental sync // starts clean). - var fc feishuCursor + var fc core.FeishuCursor nb, _ := json.Marshal(next2.ConnectorCursor) _ = json.Unmarshal(nb, &fc) for _, n := range nodes { if _, ok := fc.SpaceNodeTimes["space1"][n.NodeToken]; !ok { - t.Errorf("final cursor missing %s — incremental sync would re-fetch it", n.NodeToken) + t.Errorf("final cursor missing %s — incremental sync would re-core.Fetch it", n.NodeToken) } } } diff --git a/internal/datasource/connector/feishu/connector_golden_test.go b/internal/datasource/connector/feishu/wiki/connector_golden_test.go similarity index 68% rename from internal/datasource/connector/feishu/connector_golden_test.go rename to internal/datasource/connector/feishu/wiki/connector_golden_test.go index f6717366b..a7702210c 100644 --- a/internal/datasource/connector/feishu/connector_golden_test.go +++ b/internal/datasource/connector/feishu/wiki/connector_golden_test.go @@ -1,4 +1,4 @@ -package feishu +package wiki import ( "bytes" @@ -8,87 +8,69 @@ import ( "strings" "testing" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/types" ) // ────────────────────────────────────────────────────────────────────── // Golden end-to-end test: a single rich docx node exercising EVERY new // capability at once, driven through the real Connector.FetchAll → real -// Client → Feishu-shaped fake endpoints (blocks / sheets-v2 / bitable-v1 / +// core.Client → Feishu-shaped fake endpoints (blocks / sheets-v2 / bitable-v1 / // medias). This is the glue test the per-function unit tests can't be: // it proves the whole pipeline produces correct, ordered Markdown plus the // correctly filtered attachment sub-items. // ────────────────────────────────────────────────────────────────────── // blk constructors kept local to this file to keep the fixture readable. -func sheetBlk(id, token string) docxBlock { - return docxBlock{BlockID: id, BlockType: blockTypeSheet, Sheet: &struct { - Token string `json:"token"` - }{Token: token}} +func sheetBlk(id, token string) core.DocxBlock { + return core.DocxBlock{BlockID: id, BlockType: core.BlockTypeSheet, Sheet: &core.BlockTokenRef{Token: token}} } -func bitableBlk(id, token string) docxBlock { - return docxBlock{BlockID: id, BlockType: blockTypeBitable, Bitable: &struct { - Token string `json:"token"` - }{Token: token}} + +func bitableBlk(id, token string) core.DocxBlock { + return core.DocxBlock{BlockID: id, BlockType: core.BlockTypeBitable, Bitable: &core.BlockTokenRef{Token: token}} } -func imageBlk(id, token string) docxBlock { - return docxBlock{BlockID: id, BlockType: blockTypeImage, Image: &struct { - Token string `json:"token"` - }{Token: token}} + +func imageBlk(id, token string) core.DocxBlock { + return core.DocxBlock{BlockID: id, BlockType: core.BlockTypeImage, Image: &core.BlockTokenRef{Token: token}} } -func fileBlk(id, token, name string) docxBlock { - return docxBlock{BlockID: id, BlockType: blockTypeFile, File: &struct { - Token string `json:"token"` - Name string `json:"name"` - }{Token: token, Name: name}} + +func fileBlk(id, token, name string) core.DocxBlock { + return core.DocxBlock{BlockID: id, BlockType: core.BlockTypeFile, File: &core.BlockFileRef{Token: token, Name: name}} } // cellBlk builds a table_cell container. A real Feishu table_cell holds no // inline text of its own — its text lives in a child block, so the cell only // references the child id (see cellTextBlk). -func cellBlk(id string) docxBlock { - return docxBlock{BlockID: id, BlockType: blockTypeTableCell, Children: []string{id + "_txt"}} +func cellBlk(id string) core.DocxBlock { + return core.DocxBlock{BlockID: id, BlockType: core.BlockTypeTableCell, Children: []string{id + "_txt"}} } // cellTextBlk builds the child text block a table cell points at. -func cellTextBlk(id, text string) docxBlock { - return docxBlock{BlockID: id + "_txt", BlockType: blockTypeText, Text: txt(text)} +func cellTextBlk(id, text string) core.DocxBlock { + return core.DocxBlock{BlockID: id + "_txt", BlockType: core.BlockTypeText, Text: txt(text)} } -func tableBlk(id string, cols int, cellIDs ...string) docxBlock { - b := docxBlock{BlockID: id, BlockType: blockTypeTable} - b.Table = &struct { - Cells []string `json:"cells"` - Property *struct { - ColumnSize int `json:"column_size"` - } `json:"property"` - }{Cells: cellIDs} - b.Table.Property = &struct { - ColumnSize int `json:"column_size"` - }{ColumnSize: cols} + +func tableBlk(id string, cols int, cellIDs ...string) core.DocxBlock { + b := core.DocxBlock{BlockID: id, BlockType: core.BlockTypeTable} + b.Table = &core.BlockTable{Cells: cellIDs} + b.Table.Property = &core.BlockTableProperty{ColumnSize: cols} return b } // fakeFeishuGolden serves the full API surface a rich docx node needs. -func fakeFeishuGolden(nodes []wikiNode, docToken string, blocks []docxBlock, - mediaByToken map[string][]byte) (*httptest.Server, *Config) { +func fakeFeishuGolden(nodes []core.WikiNode, docToken string, blocks []core.DocxBlock, + mediaByToken map[string][]byte, +) (*httptest.Server, *core.Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) + writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{apiResponse: apiResponse{Code: 0}, Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}}}) + writeJSON(w, core.WikiSpaceListResponse{ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}}) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiNodeListResponse{apiResponse: apiResponse{Code: 0}, Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: nodes}}) + writeJSON(w, core.WikiNodeListResponse{ApiResponse: core.ApiResponse{Code: 0}, Data: core.WikiNodeListData{Items: nodes}}) }) // docx blocks — paginated across two pages to exercise the paging glue. @@ -104,11 +86,7 @@ func fakeFeishuGolden(nodes []wikiNode, docToken string, blocks []docxBlock, if !hasMore { nextTok = "" } - writeJSON(w, docxBlocksResponse{apiResponse: apiResponse{Code: 0}, Data: struct { - Items []docxBlock `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: page, HasMore: hasMore, PageToken: nextTok}}) + writeJSON(w, core.DocxBlocksResponse{ApiResponse: core.ApiResponse{Code: 0}, Data: core.DocxBlocksData{Items: page, HasMore: hasMore, PageToken: nextTok}}) }) // sheets-v2 values: sht_spread_0 → spreadsheet "sht_spread", sheet "0". @@ -148,26 +126,27 @@ func fakeFeishuGolden(nodes []wikiNode, docToken string, blocks []docxBlock, }) ts := httptest.NewServer(mux) - return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} + return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} } func TestGolden_RichDocxAllCapabilities(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const docToken = "obj-golden" - bigPDF := bytes.Repeat([]byte("A"), minAttachmentBytes+512) // kept - tinyPDF := bytes.Repeat([]byte("B"), 100) // whitelisted but too small → dropped + bigPDF := bytes.Repeat([]byte("A"), core.MinAttachmentBytes+512) // kept + tinyPDF := bytes.Repeat([]byte("B"), 100) // whitelisted but too small → dropped - blocks := []docxBlock{ - {BlockID: "root", BlockType: blockTypePage}, - {BlockID: "h1", BlockType: blockTypeHeading1, Heading1: txt("季度报告")}, - {BlockID: "p1", BlockType: blockTypeText, Text: txt("本季度概览。")}, - {BlockID: "h2", BlockType: blockTypeHeading1 + 1, Heading2: txt("关键指标")}, - {BlockID: "b1", BlockType: blockTypeBullet, Bullet: txt("收入增长")}, - {BlockID: "o1", BlockType: blockTypeOrdered, Ordered: txt("第一步立项")}, - {BlockID: "code1", BlockType: blockTypeCode, Code: txt("SELECT 1")}, - {BlockID: "q1", BlockType: blockTypeQuote, Quote: txt("重要提示")}, - {BlockID: "todo1", BlockType: blockTypeTodo, Todo: txt("完成复盘")}, - {BlockID: "call1", BlockType: blockTypeCallout, Callout: txt("注意风险")}, - {BlockID: "div1", BlockType: blockTypeDivider}, + blocks := []core.DocxBlock{ + {BlockID: "root", BlockType: core.BlockTypePage}, + {BlockID: "h1", BlockType: core.BlockTypeHeading1, Heading1: txt("季度报告")}, + {BlockID: "p1", BlockType: core.BlockTypeText, Text: txt("本季度概览。")}, + {BlockID: "h2", BlockType: core.BlockTypeHeading1 + 1, Heading2: txt("关键指标")}, + {BlockID: "b1", BlockType: core.BlockTypeBullet, Bullet: txt("收入增长")}, + {BlockID: "o1", BlockType: core.BlockTypeOrdered, Ordered: txt("第一步立项")}, + {BlockID: "code1", BlockType: core.BlockTypeCode, Code: txt("SELECT 1")}, + {BlockID: "q1", BlockType: core.BlockTypeQuote, Quote: txt("重要提示")}, + {BlockID: "todo1", BlockType: core.BlockTypeTodo, Todo: txt("完成复盘")}, + {BlockID: "call1", BlockType: core.BlockTypeCallout, Callout: txt("注意风险")}, + {BlockID: "div1", BlockType: core.BlockTypeDivider}, tableBlk("tbl1", 2, "c1", "c2", "c3", "c4"), sheetBlk("sh1", "sht_spread_0"), bitableBlk("bt1", "bascApp_tblMain"), @@ -175,14 +154,14 @@ func TestGolden_RichDocxAllCapabilities(t *testing.T) { fileBlk("fbig", "tok-big", "手册.pdf"), fileBlk("flogo", "tok-logo", "logo.png"), // non-whitelisted ext → no sub-item fileBlk("fsmall", "tok-small", "small.pdf"), // whitelisted but too small → no sub-item - // table cells (containers) + their child text blocks — both are skipped by + // table cells (containers) + their child text blocks — both are core.Skipped by // the main loop and read by the table renderer via the cells' Children. cellBlk("c1"), cellBlk("c2"), cellBlk("c3"), cellBlk("c4"), cellTextBlk("c1", "列A"), cellTextBlk("c2", "列B"), cellTextBlk("c3", "1"), cellTextBlk("c4", "2"), } - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: "nt-golden", ObjToken: docToken, ObjType: "docx", Title: "季度报告文档", NodeEditTime: "1711468800", }} @@ -191,7 +170,7 @@ func TestGolden_RichDocxAllCapabilities(t *testing.T) { ts, cfg := fakeFeishuGolden(nodes, docToken, blocks, media) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll error: %v", err) diff --git a/internal/datasource/connector/feishu/connector_realapi_test.go b/internal/datasource/connector/feishu/wiki/connector_realapi_test.go similarity index 91% rename from internal/datasource/connector/feishu/connector_realapi_test.go rename to internal/datasource/connector/feishu/wiki/connector_realapi_test.go index 1bbae552f..7b4cd1377 100644 --- a/internal/datasource/connector/feishu/connector_realapi_test.go +++ b/internal/datasource/connector/feishu/wiki/connector_realapi_test.go @@ -16,7 +16,7 @@ // wiki space but never creates, edits or deletes anything. It whitelists // *.feishu.cn / *.larksuite.com for SSRF in-process so it works from a dev // machine behind a fake-ip proxy without any production code change. -package feishu +package wiki import ( "context" @@ -25,6 +25,7 @@ import ( "strings" "testing" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/types" "github.com/Tencent/WeKnora/internal/utils" ) @@ -74,7 +75,7 @@ func nodeTimes(t *testing.T, cur *types.SyncCursor) map[string]map[string]string if cur == nil { return nil } - var fc feishuCursor + var fc core.FeishuCursor b, _ := json.Marshal(cur.ConnectorCursor) _ = json.Unmarshal(b, &fc) return fc.SpaceNodeTimes @@ -98,7 +99,7 @@ func TestRealAPI_ListSpaces(t *testing.T) { } utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com") - client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")}) + client := core.NewClient(&core.Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")}) spaces, err := client.ListWikiSpaces(context.Background()) if err != nil { t.Fatalf("ListWikiSpaces: %v", err) @@ -133,7 +134,7 @@ func TestRealAPI_FetchStreamResumeConverges(t *testing.T) { }, ResourceIDs: []string{spaceID}, } - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) // ---- Pass 1: full sync against the real space. h1 := &collectHandler{} @@ -151,7 +152,7 @@ func TestRealAPI_FetchStreamResumeConverges(t *testing.T) { // also ingests child items whose external_id is "#file#..." or // "#image#...". Those are not wiki nodes: they ride their parent node's // edit-time and intentionally carry no independent cursor entry (the parent's - // edit-time gates re-fetch of the whole subtree). Only node-level ids are + // edit-time gates re-core.Fetch of the whole subtree). Only node-level ids are // required to be present. nt1 := nodeTimes(t, cur1)[spaceID] for _, id := range h1.ingested { @@ -173,7 +174,7 @@ func TestRealAPI_FetchStreamResumeConverges(t *testing.T) { } t.Logf("pass 2: re-ingested=%d (want 0)", len(h2.ingested)) if len(h2.ingested) != 0 { - t.Errorf("pass 2 re-ingested %v — real edit-time is unstable or skip logic is broken", h2.ingested) + t.Errorf("pass 2 re-ingested %v — real edit-time is unstable or core.Skip logic is broken", h2.ingested) } if got := countNodes(nodeTimes(t, cur2)); got != countNodes(nt1AsMap(nt1)) { t.Logf("note: cursor node count changed between passes (%d→%d) — space may have been edited concurrently", countNodes(nt1AsMap(nt1)), got) @@ -185,9 +186,9 @@ func TestRealAPI_FetchStreamResumeConverges(t *testing.T) { t.Logf("only %d doc(s); skipping interrupt/resume convergence sub-test (needs >=2)", n) return } - prevN := feishuStreamCheckpointInterval - feishuStreamCheckpointInterval = 1 // checkpoint every node so resume is precise - defer func() { feishuStreamCheckpointInterval = prevN }() + prevN := core.FeishuStreamCheckpointInterval + core.FeishuStreamCheckpointInterval = 1 // Checkpoint every node so resume is precise + defer func() { core.FeishuStreamCheckpointInterval = prevN }() ctx3, cancel3 := context.WithCancel(context.Background()) h3 := &collectHandler{cancelAfter: 1, cancel: cancel3} // abort after 1 success @@ -197,12 +198,12 @@ func TestRealAPI_FetchStreamResumeConverges(t *testing.T) { t.Fatalf("pass 3 expected an abort error from the simulated timeout") } if len(h3.checkpoints) == 0 { - t.Fatalf("pass 3 wrote no checkpoint — resume would restart from scratch") + t.Fatalf("pass 3 wrote no Checkpoint — resume would restart from scratch") } persisted := h3.checkpoints[len(h3.checkpoints)-1] t.Logf("pass 3: ingested=%d before abort, persisted cursor nodes=%d", len(h3.ingested), countNodes(nodeTimes(t, persisted))) - // Resume from the persisted checkpoint; must converge to full coverage. + // Resume from the persisted Checkpoint; must converge to full coverage. h4 := &collectHandler{} _, err = c.FetchStream(context.Background(), cfg, persisted, h4) if err != nil { diff --git a/internal/datasource/connector/feishu/connector_seed_test.go b/internal/datasource/connector/feishu/wiki/connector_seed_test.go similarity index 91% rename from internal/datasource/connector/feishu/connector_seed_test.go rename to internal/datasource/connector/feishu/wiki/connector_seed_test.go index 0d23cb990..ad7d26571 100644 --- a/internal/datasource/connector/feishu/connector_seed_test.go +++ b/internal/datasource/connector/feishu/wiki/connector_seed_test.go @@ -8,6 +8,7 @@ // and delete them afterwards. Requires app write scopes: // - wiki:wiki (create wiki nodes) // - docx:document (write docx content) +// // plus the app being a member of the space. // // Run: @@ -16,7 +17,7 @@ // FEISHU_APP_ID=... FEISHU_APP_SECRET=... FEISHU_TEST_SPACE_ID=... FEISHU_SEED_COUNT=3 \ // go test -tags feishu_integration -run TestRealAPI_SeedDocs -v \ // ./internal/datasource/connector/feishu/ -package feishu +package wiki import ( "context" @@ -25,6 +26,7 @@ import ( "strconv" "testing" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/utils" ) @@ -52,7 +54,7 @@ func TestRealAPI_CreateSpace(t *testing.T) { t.Skip("set FEISHU_APP_ID, FEISHU_APP_SECRET to run") } utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com") - client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")}) + client := core.NewClient(&core.Config{AppID: appID, AppSecret: appSecret, BaseURL: os.Getenv("FEISHU_BASE_URL")}) var resp struct { Code int `json:"code"` @@ -64,7 +66,7 @@ func TestRealAPI_CreateSpace(t *testing.T) { } `json:"space"` } `json:"data"` } - err := client.doRequest(context.Background(), "POST", "/open-apis/wiki/v2/spaces", + err := client.DoRequest(context.Background(), "POST", "/open-apis/wiki/v2/spaces", map[string]interface{}{ "name": "WeKnora Resilience Test KB", "description": "Auto-created for connector resilience testing (#2136). Safe to delete.", @@ -95,7 +97,7 @@ func TestRealAPI_SeedDocs(t *testing.T) { utils.SetSSRFWhitelistFromRaw("*.feishu.cn,*.larksuite.com") baseURL := os.Getenv("FEISHU_BASE_URL") - client := NewClient(&Config{AppID: appID, AppSecret: appSecret, BaseURL: baseURL}) + client := core.NewClient(&core.Config{AppID: appID, AppSecret: appSecret, BaseURL: baseURL}) ctx := context.Background() for i := 1; i <= count; i++ { @@ -103,7 +105,7 @@ func TestRealAPI_SeedDocs(t *testing.T) { // 1) Create a docx wiki node. var created wikiNodeCreateResp - err := client.doRequest(ctx, "POST", + err := client.DoRequest(ctx, "POST", fmt.Sprintf("/open-apis/wiki/v2/spaces/%s/nodes", spaceID), map[string]interface{}{ "obj_type": "docx", @@ -123,7 +125,7 @@ func TestRealAPI_SeedDocs(t *testing.T) { Code int `json:"code"` Msg string `json:"msg"` } - berr := client.doRequest(ctx, "POST", + berr := client.DoRequest(ctx, "POST", fmt.Sprintf("/open-apis/docx/v1/documents/%s/blocks/%s/children", docID, docID), map[string]interface{}{ "index": 0, diff --git a/internal/datasource/connector/feishu/connector_stream_test.go b/internal/datasource/connector/feishu/wiki/connector_stream_test.go similarity index 75% rename from internal/datasource/connector/feishu/connector_stream_test.go rename to internal/datasource/connector/feishu/wiki/connector_stream_test.go index 5dcf82086..16f087683 100644 --- a/internal/datasource/connector/feishu/connector_stream_test.go +++ b/internal/datasource/connector/feishu/wiki/connector_stream_test.go @@ -1,4 +1,4 @@ -package feishu +package wiki import ( "bytes" @@ -10,58 +10,51 @@ import ( "strings" "testing" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/types" ) // fakeFeishuFailingExport serves the auth/spaces/nodes endpoints normally but // fails every document export (code != 0), so fetchNodeContent returns an error // for each supported node — modelling a rate-limited / broken export. -func fakeFeishuFailingExport(nodes []wikiNode) (*httptest.Server, *Config) { +func fakeFeishuFailingExport(nodes []core.WikiNode) (*httptest.Server, *core.Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) + writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, + writeJSON(w, core.WikiSpaceListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiNodeListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: nodes}, + writeJSON(w, core.WikiNodeListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeListData{Items: nodes}, }) }) // Export creation fails for every document. mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, apiResponse{Code: 1, Msg: "export unavailable"}) + writeJSON(w, core.ApiResponse{Code: 1, Msg: "export unavailable"}) }) ts := httptest.NewServer(mux) - return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} + return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} } -// A node whose fetch fails must NOT have its new edit time recorded in the +// A node whose core.Fetch fails must NOT have its new edit time recorded in the // returned cursor: recording it would make the next sync's unchanged fast-path -// skip it forever, silently dropping a document on a transient export failure +// core.Skip it forever, silently dropping a document on a transient export failure // (Tencent/WeKnora#2136). With a prior edit time known, the prior value is // retained so prev != current next run and the node is retried. func TestFetchStream_FailedFetchRetainsPriorCursor(t *testing.T) { - nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}} + nodes := []core.WikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}} ts, cfg := fakeFeishuFailingExport(nodes) defer ts.Close() cursor := makeStreamCursor(t, map[string]map[string]string{"space1": {"nt1": "50"}}) // prior, older - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h) if err != nil { @@ -73,33 +66,33 @@ func TestFetchStream_FailedFetchRetainsPriorCursor(t *testing.T) { t.Fatalf("expected 1 emitted failure item with error metadata, got %+v", h.emitted) } - var fc feishuCursor + var fc core.FeishuCursor b, _ := json.Marshal(next.ConnectorCursor) _ = json.Unmarshal(b, &fc) got := fc.SpaceNodeTimes["space1"]["nt1"] if got == "100" { - t.Fatalf("failed node advanced to current edit time %q — it will be skipped forever", got) + t.Fatalf("failed node advanced to current edit time %q — it will be core.Skipped forever", got) } if got != "50" { t.Errorf("failed node cursor = %q, want prior value \"50\" (retry next run)", got) } } -// With no prior cursor entry, a failed fetch must leave the node out of the +// With no prior cursor entry, a failed core.Fetch must leave the node out of the // returned cursor entirely, so the next run treats it as new and retries it. func TestFetchStream_FailedFetchNoPriorOmitsFromCursor(t *testing.T) { - nodes := []wikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}} + nodes := []core.WikiNode{{NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}} ts, cfg := fakeFeishuFailingExport(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h) if err != nil { t.Fatalf("FetchStream() error: %v", err) } - var fc feishuCursor + var fc core.FeishuCursor b, _ := json.Marshal(next.ConnectorCursor) _ = json.Unmarshal(b, &fc) if v, ok := fc.SpaceNodeTimes["space1"]["nt1"]; ok { @@ -107,14 +100,14 @@ func TestFetchStream_FailedFetchNoPriorOmitsFromCursor(t *testing.T) { } } -// recordingHandler captures the items and checkpoints a streaming fetch emits. +// recordingHandler captures the items and checkpoints a streaming core.Fetch emits. // Checkpoints are snapshotted (JSON-encoded) at call time — mirroring the // service, which serializes the cursor synchronously inside Checkpoint — so the // test observes the cursor state as it was when Checkpoint was called, not the // connector's later-mutated map. type recordingHandler struct { emitted []types.FetchedItem - checkpoints []feishuCursor + checkpoints []core.FeishuCursor emitErr func(item types.FetchedItem) error } @@ -129,7 +122,7 @@ func (h *recordingHandler) Emit(ctx context.Context, item types.FetchedItem) err } func (h *recordingHandler) Checkpoint(ctx context.Context, cursor *types.SyncCursor) error { - var fc feishuCursor + var fc core.FeishuCursor b, _ := json.Marshal(cursor.ConnectorCursor) _ = json.Unmarshal(b, &fc) h.checkpoints = append(h.checkpoints, fc) @@ -138,7 +131,7 @@ func (h *recordingHandler) Checkpoint(ctx context.Context, cursor *types.SyncCur func makeStreamCursor(t *testing.T, spaceNodeTimes map[string]map[string]string) *types.SyncCursor { t.Helper() - prev := feishuCursor{SpaceNodeTimes: spaceNodeTimes} + prev := core.FeishuCursor{SpaceNodeTimes: spaceNodeTimes} b, _ := json.Marshal(prev) var m map[string]interface{} if err := json.Unmarshal(b, &m); err != nil { @@ -151,7 +144,7 @@ func makeStreamCursor(t *testing.T, spaceNodeTimes map[string]map[string]string) // types, and the returned cursor records the edit time of every discovered node // (so the next incremental run can detect changes). func TestFetchStream_EmitsSupportedSkipsUnsupported(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "mindnote", Title: "Brain", ObjEditTime: "200"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"}, @@ -159,7 +152,7 @@ func TestFetchStream_EmitsSupportedSkipsUnsupported(t *testing.T) { ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} next, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h) if err != nil { @@ -173,7 +166,7 @@ func TestFetchStream_EmitsSupportedSkipsUnsupported(t *testing.T) { t.Errorf("emitted ids = %q,%q; want nt1,nt3", h.emitted[0].ExternalID, h.emitted[1].ExternalID) } - var fc feishuCursor + var fc core.FeishuCursor b, _ := json.Marshal(next.ConnectorCursor) _ = json.Unmarshal(b, &fc) times := fc.SpaceNodeTimes["space1"] @@ -186,10 +179,10 @@ func TestFetchStream_EmitsSupportedSkipsUnsupported(t *testing.T) { // When a cursor already records a node at its current edit time, that node is // unchanged and must not be re-emitted; only new/changed nodes stream through. -// This is the resume/incremental-skip behavior that lets a timed-out sync +// This is the resume/incremental-core.Skip behavior that lets a timed-out sync // converge across retries instead of re-exporting everything. func TestFetchStream_SkipsUnchangedNodesFromCursor(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"}, } @@ -200,7 +193,7 @@ func TestFetchStream_SkipsUnchangedNodesFromCursor(t *testing.T) { "space1": {"nt1": "100"}, // nt1 unchanged; nt3 unknown → new }) - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), cursor, h); err != nil { t.Fatalf("FetchStream() error: %v", err) @@ -215,33 +208,33 @@ func TestFetchStream_SkipsUnchangedNodesFromCursor(t *testing.T) { } // Checkpoints must persist progress at page boundaries so a crash mid-sync -// resumes from the last checkpoint. With the interval set to 1, each emitted -// item triggers a checkpoint, and the first checkpoint must already contain the +// resumes from the last Checkpoint. With the interval set to 1, each emitted +// item triggers a Checkpoint, and the first Checkpoint must already contain the // first node's edit time. func TestFetchStream_CheckpointsProgress(t *testing.T) { - prev := feishuStreamCheckpointInterval - feishuStreamCheckpointInterval = 1 - defer func() { feishuStreamCheckpointInterval = prev }() + prev := core.FeishuStreamCheckpointInterval + core.FeishuStreamCheckpointInterval = 1 + defer func() { core.FeishuStreamCheckpointInterval = prev }() - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil { t.Fatalf("FetchStream() error: %v", err) } if len(h.checkpoints) == 0 { - t.Fatalf("expected at least one checkpoint") + t.Fatalf("expected at least one Checkpoint") } first := h.checkpoints[0] if _, ok := first.SpaceNodeTimes["space1"]["nt1"]; !ok { - t.Errorf("first checkpoint missing nt1 progress: %+v", first.SpaceNodeTimes) + t.Errorf("first Checkpoint missing nt1 progress: %+v", first.SpaceNodeTimes) } } @@ -250,25 +243,25 @@ func TestFetchStream_CheckpointsProgress(t *testing.T) { // exports reaches the 2h task timeout having never checkpointed, and resumes // from scratch forever — exactly the #2136 "never fully syncs" case. With the // node interval effectively disabled and the time interval at 0, every -// processed node must still produce a checkpoint. +// processed node must still produce a Checkpoint. func TestFetchStream_CheckpointsOnElapsedTime(t *testing.T) { - prevN := feishuStreamCheckpointInterval - prevT := feishuStreamCheckpointMaxInterval - feishuStreamCheckpointInterval = 1 << 30 // never fires by count - feishuStreamCheckpointMaxInterval = 0 // fires by elapsed time every node + prevN := core.FeishuStreamCheckpointInterval + prevT := core.FeishuStreamCheckpointMaxInterval + core.FeishuStreamCheckpointInterval = 1 << 30 // never fires by count + core.FeishuStreamCheckpointMaxInterval = 0 // fires by elapsed time every node defer func() { - feishuStreamCheckpointInterval = prevN - feishuStreamCheckpointMaxInterval = prevT + core.FeishuStreamCheckpointInterval = prevN + core.FeishuStreamCheckpointMaxInterval = prevT }() - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} if _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h); err != nil { t.Fatalf("FetchStream() error: %v", err) @@ -282,7 +275,7 @@ func TestFetchStream_CheckpointsOnElapsedTime(t *testing.T) { // error and stop fetching further nodes (the sync is failing; do not burn API // budget on the rest of the tree). func TestFetchStream_EmitErrorAborts(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", ObjEditTime: "100"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "docx", Title: "Doc3", ObjEditTime: "300"}, } @@ -290,14 +283,14 @@ func TestFetchStream_EmitErrorAborts(t *testing.T) { defer ts.Close() boom := errors.New("ingest failed") - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{emitErr: func(item types.FetchedItem) error { return boom }} _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h) if !errors.Is(err, boom) { t.Fatalf("FetchStream() error = %v, want %v", err, boom) } if len(h.emitted) != 0 { - t.Errorf("emitted %d items, want 0 (aborted on first emit)", len(h.emitted)) + t.Errorf("emitted %d items, want 0 (aborted on first Emit)", len(h.emitted)) } } @@ -307,21 +300,22 @@ func TestFetchStream_EmitErrorAborts(t *testing.T) { // TestFetchStream_DocxMultiItem is a stream-level integration test proving that a // docx node fans out to a main Markdown item + an attachment item through the real -// FetchStream → fetchNodeContent → fetchDocxWithBlocks path. The fake server +// FetchStream → fetchNodeContent → core.FetchDocxWithBlocks path. The fake server // serves the blocks API (one text block + one file block) and the drive download; // no export endpoint is registered, so any accidental fall-through to the export -// path would 404 and surface as a fetch error. +// path would 404 and surface as a core.Fetch error. func TestFetchStream_DocxMultiItem(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-blocks" objToken = "obj-blocks" attToken = "ft-stream-att" attName = "slides.pdf" ) - // Attachment content must exceed minAttachmentBytes (2 KiB). - attContent := bytes.Repeat([]byte("a"), minAttachmentBytes+1) + // Attachment content must exceed core.MinAttachmentBytes (2 KiB). + attContent := bytes.Repeat([]byte("a"), core.MinAttachmentBytes+1) - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", @@ -331,7 +325,7 @@ func TestFetchStream_DocxMultiItem(t *testing.T) { ts, cfg := fakeFeishuWithBlocks(nodes, objToken, attToken, attName, attContent) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h) if err != nil { @@ -365,35 +359,27 @@ func TestFetchStream_DocxMultiItem(t *testing.T) { // fakeFeishuWithBlocksFallback returns a server where the blocks API returns HTTP 500 // (simulating a permission/scope error) and the export task path returns success. -// This wires the fallback path: fetchDocxWithBlocks → blocks error → fetchViaExport. -func fakeFeishuWithBlocksFallback(nodes []wikiNode, docToken string) (*httptest.Server, *Config) { +// This wires the fallback path: core.FetchDocxWithBlocks → blocks error → fetchViaExport. +func fakeFeishuWithBlocksFallback(nodes []core.WikiNode, docToken string) (*httptest.Server, *core.Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{ - apiResponse: apiResponse{Code: 0}, + writeJSON(w, core.TokenResponse{ + ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, + writeJSON(w, core.WikiSpaceListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiNodeListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: nodes}, + writeJSON(w, core.WikiNodeListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeListData{Items: nodes}, }) }) @@ -407,11 +393,9 @@ func fakeFeishuWithBlocksFallback(nodes []wikiNode, docToken string) (*httptest. // Export task creation → returns ticket. mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPost { - writeJSON(w, exportTaskCreateResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Ticket string `json:"ticket"` - }{Ticket: "ticket-fb"}, + writeJSON(w, core.ExportTaskCreateResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskCreateData{Ticket: "ticket-fb"}, }) return } @@ -419,24 +403,10 @@ func fakeFeishuWithBlocksFallback(nodes []wikiNode, docToken string) (*httptest. }) // Export task status polling. mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-fb", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, exportTaskStatusResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Result struct { - FileToken string `json:"file_token"` - FileSize int64 `json:"file_size"` - JobStatus int `json:"job_status"` - JobErrorMsg string `json:"job_error_msg"` - FileName string `json:"file_name"` - } `json:"result"` - }{ - Result: struct { - FileToken string `json:"file_token"` - FileSize int64 `json:"file_size"` - JobStatus int `json:"job_status"` - JobErrorMsg string `json:"job_error_msg"` - FileName string `json:"file_name"` - }{ + writeJSON(w, core.ExportTaskStatusResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskStatusData{ + Result: core.ExportTaskResult{ FileToken: "ft-export-fallback", FileSize: 512, JobStatus: 0, // done @@ -462,7 +432,7 @@ func fakeFeishuWithBlocksFallback(nodes []wikiNode, docToken string) (*httptest. }) ts := httptest.NewServer(mux) - return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} + return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} } // TestFetchStream_DocxBlocksFallback proves that when the blocks API returns HTTP 500 @@ -471,11 +441,12 @@ func fakeFeishuWithBlocksFallback(nodes []wikiNode, docToken string) (*httptest. // "application/octet-stream", not "text/markdown". No hard failure occurs — // the sync completes successfully with the exported binary. func TestFetchStream_DocxBlocksFallback(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-fallback" objToken = "obj-fallback" ) - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", @@ -485,14 +456,14 @@ func TestFetchStream_DocxBlocksFallback(t *testing.T) { ts, cfg := fakeFeishuWithBlocksFallback(nodes, objToken) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) h := &recordingHandler{} _, err := c.FetchStream(context.Background(), makeConfig(cfg, []string{"space1"}), nil, h) if err != nil { t.Fatalf("FetchStream() error: %v", err) } - // The fallback path must emit exactly one item (exported binary, not multi-item). + // The fallback path must Emit exactly one item (exported binary, not multi-item). if len(h.emitted) != 1 { t.Fatalf("expected 1 emitted item (export fallback), got %d: %+v", len(h.emitted), h.emitted) } diff --git a/internal/datasource/connector/feishu/connector_test.go b/internal/datasource/connector/feishu/wiki/connector_test.go similarity index 79% rename from internal/datasource/connector/feishu/connector_test.go rename to internal/datasource/connector/feishu/wiki/connector_test.go index e7e672543..7d52a118c 100644 --- a/internal/datasource/connector/feishu/connector_test.go +++ b/internal/datasource/connector/feishu/wiki/connector_test.go @@ -1,4 +1,4 @@ -package feishu +package wiki import ( "bytes" @@ -14,13 +14,14 @@ import ( "time" "unicode/utf8" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/logger" "github.com/Tencent/WeKnora/internal/types" secutils "github.com/Tencent/WeKnora/internal/utils" ) func TestMain(m *testing.M) { - os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost") + os.Setenv("SSRF_WHITELIST", "127.0.0.1,localhost,open.feishu.cn,open.larksuite.com") secutils.ResetSSRFWhitelistForTest() os.Exit(m.Run()) } @@ -30,14 +31,14 @@ func TestMain(m *testing.M) { // ────────────────────────────────────────────────────────────────────── // fakeFeishu builds an httptest.Server that emulates the relevant Feishu APIs. -// It returns the server and a Config pointing at it. -func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { +// It returns the server and a core.Config pointing at it. +func fakeFeishu(nodes []core.WikiNode) (*httptest.Server, *core.Config) { mux := http.NewServeMux() // --- auth --- mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{ - apiResponse: apiResponse{Code: 0}, + writeJSON(w, core.TokenResponse{ + ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) @@ -45,14 +46,10 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { // --- wiki spaces --- mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ - Items: []wikiSpace{ + writeJSON(w, core.WikiSpaceListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiSpaceListData{ + Items: []core.WikiSpace{ {SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"}, }, }, @@ -61,13 +58,9 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { // --- wiki nodes (top-level only for simplicity) --- mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiNodeListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ + writeJSON(w, core.WikiNodeListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeListData{ Items: nodes, }, }) @@ -76,33 +69,17 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { // --- export task: create --- mux.HandleFunc("/open-apis/drive/v1/export_tasks", func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodPost { - writeJSON(w, exportTaskCreateResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Ticket string `json:"ticket"` - }{Ticket: "ticket-123"}, + writeJSON(w, core.ExportTaskCreateResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskCreateData{Ticket: "ticket-123"}, }) return } // GET /open-apis/drive/v1/export_tasks/ticket-123 - writeJSON(w, exportTaskStatusResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Result struct { - FileToken string `json:"file_token"` - FileSize int64 `json:"file_size"` - JobStatus int `json:"job_status"` - JobErrorMsg string `json:"job_error_msg"` - FileName string `json:"file_name"` - } `json:"result"` - }{ - Result: struct { - FileToken string `json:"file_token"` - FileSize int64 `json:"file_size"` - JobStatus int `json:"job_status"` - JobErrorMsg string `json:"job_error_msg"` - FileName string `json:"file_name"` - }{ + writeJSON(w, core.ExportTaskStatusResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskStatusData{ + Result: core.ExportTaskResult{ FileToken: "ft-abc", FileSize: 100, JobStatus: 0, // success @@ -114,24 +91,10 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { // --- export task: status polling (pattern match with ticket) --- mux.HandleFunc("/open-apis/drive/v1/export_tasks/ticket-123", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, exportTaskStatusResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Result struct { - FileToken string `json:"file_token"` - FileSize int64 `json:"file_size"` - JobStatus int `json:"job_status"` - JobErrorMsg string `json:"job_error_msg"` - FileName string `json:"file_name"` - } `json:"result"` - }{ - Result: struct { - FileToken string `json:"file_token"` - FileSize int64 `json:"file_size"` - JobStatus int `json:"job_status"` - JobErrorMsg string `json:"job_error_msg"` - FileName string `json:"file_name"` - }{ + writeJSON(w, core.ExportTaskStatusResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.ExportTaskStatusData{ + Result: core.ExportTaskResult{ FileToken: "ft-abc", FileSize: 100, JobStatus: 0, @@ -158,7 +121,7 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { }) ts := httptest.NewServer(mux) - cfg := &Config{ + cfg := &core.Config{ AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL, @@ -166,13 +129,13 @@ func fakeFeishu(nodes []wikiNode) (*httptest.Server, *Config) { return ts, cfg } -func fakeFeishuWithChildFailure(topNodes []wikiNode, failingParentToken string) (*httptest.Server, *Config) { +func fakeFeishuWithChildFailure(topNodes []core.WikiNode, failingParentToken string) (*httptest.Server, *core.Config) { return fakeFeishuHierarchy(topNodes, nil, failingParentToken) } -func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, failingParentToken string) (*httptest.Server, *Config) { +func fakeFeishuHierarchy(topNodes []core.WikiNode, childNodes map[string][]core.WikiNode, failingParentToken string) (*httptest.Server, *core.Config) { mux := http.NewServeMux() - nodeByToken := make(map[string]wikiNode) + nodeByToken := make(map[string]core.WikiNode) for _, node := range topNodes { node.SpaceID = "space1" nodeByToken[node.NodeToken] = node @@ -188,22 +151,18 @@ func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, } mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{ - apiResponse: apiResponse{Code: 0}, + writeJSON(w, core.TokenResponse{ + ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ - Items: []wikiSpace{ + writeJSON(w, core.WikiSpaceListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiSpaceListData{ + Items: []core.WikiSpace{ {SpaceID: "space1", Name: "Test Space", Description: "desc", Visibility: "public"}, }, }, @@ -229,13 +188,9 @@ func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, } } } - writeJSON(w, wikiNodeListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ + writeJSON(w, core.WikiNodeListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeListData{ Items: nodes, }, }) @@ -245,16 +200,14 @@ func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, nodeToken := r.URL.Query().Get("token") node, ok := nodeByToken[nodeToken] if !ok { - writeJSON(w, wikiNodeInfoResponse{ - apiResponse: apiResponse{Code: 1663, Msg: "node not found"}, + writeJSON(w, core.WikiNodeInfoResponse{ + ApiResponse: core.ApiResponse{Code: 1663, Msg: "node not found"}, }) return } - writeJSON(w, wikiNodeInfoResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Node wikiNode `json:"node"` - }{Node: node}, + writeJSON(w, core.WikiNodeInfoResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeInfoData{Node: node}, }) }) @@ -268,7 +221,7 @@ func fakeFeishuHierarchy(topNodes []wikiNode, childNodes map[string][]wikiNode, }) ts := httptest.NewServer(mux) - cfg := &Config{ + cfg := &core.Config{ AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL, @@ -281,7 +234,7 @@ func writeJSON(w http.ResponseWriter, v interface{}) { json.NewEncoder(w).Encode(v) } -func makeConfig(cfg *Config, resourceIDs []string) *types.DataSourceConfig { +func makeConfig(cfg *core.Config, resourceIDs []string) *types.DataSourceConfig { creds := map[string]interface{}{ "app_id": cfg.AppID, "app_secret": cfg.AppSecret, @@ -316,9 +269,9 @@ func TestIsSupportedDocType(t *testing.T) { for _, tt := range tests { t.Run(tt.objType, func(t *testing.T) { - got := isSupportedDocType(tt.objType) + got := core.IsSupportedDocType(tt.objType) if got != tt.expected { - t.Errorf("isSupportedDocType(%q) = %v, want %v", tt.objType, got, tt.expected) + t.Errorf("core.IsSupportedDocType(%q) = %v, want %v", tt.objType, got, tt.expected) } }) } @@ -338,9 +291,9 @@ func TestSanitizeFileName(t *testing.T) { for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - got := sanitizeFileName(tt.input) + got := core.SanitizeFileName(tt.input) if got != tt.expected { - t.Errorf("sanitizeFileName(%q) = %q, want %q", tt.input, got, tt.expected) + t.Errorf("core.SanitizeFileName(%q) = %q, want %q", tt.input, got, tt.expected) } }) } @@ -350,9 +303,9 @@ func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) { // Each 测 is 3 bytes; raw byte truncation at 200 would split a rune and // produce invalid UTF-8 that downstream filename validation rejects. long := strings.Repeat("测试", 100) - got := sanitizeFileName(long) + got := core.SanitizeFileName(long) if !utf8.ValidString(got) { - t.Fatalf("sanitizeFileName produced invalid UTF-8: %q", got) + t.Fatalf("core.SanitizeFileName produced invalid UTF-8: %q", got) } if len(got) > 200 { t.Errorf("len = %d, want ≤ 200", len(got)) @@ -367,7 +320,7 @@ func TestSanitizeFileName_PreservesExtensionOnTruncation(t *testing.T) { // Truncation must keep the ".pdf" suffix — downstream file-type validation // classifies by extension, so a chopped extension would reject the file. long := strings.Repeat("报告", 150) + ".pdf" // 150*6 bytes + ".pdf" - got := sanitizeFileName(long) + got := core.SanitizeFileName(long) if !utf8.ValidString(got) { t.Fatalf("produced invalid UTF-8: %q", got) } @@ -380,7 +333,7 @@ func TestSanitizeFileName_PreservesExtensionOnTruncation(t *testing.T) { } func TestParseFeishuTimestamp(t *testing.T) { - ts := parseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC + ts := core.ParseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC if ts.IsZero() { t.Fatal("expected non-zero time") } @@ -388,23 +341,23 @@ func TestParseFeishuTimestamp(t *testing.T) { t.Errorf("unexpected unix = %d", ts.Unix()) } - if !parseFeishuTimestamp("").IsZero() { + if !core.ParseFeishuTimestamp("").IsZero() { t.Error("expected zero time for empty string") } - if !parseFeishuTimestamp("invalid").IsZero() { + if !core.ParseFeishuTimestamp("invalid").IsZero() { t.Error("expected zero time for invalid string") } } func TestParseFeishuConfig(t *testing.T) { t.Run("valid", func(t *testing.T) { - cfg, err := parseFeishuConfig(&types.DataSourceConfig{ + cfg, err := core.ParseFeishuConfig(&types.DataSourceConfig{ Credentials: map[string]interface{}{ "app_id": "id1", "app_secret": "sec1", "base_url": "https://open.feishu.cn", }, - }, RegionFeishu) + }, core.RegionFeishu) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -414,19 +367,19 @@ func TestParseFeishuConfig(t *testing.T) { }) t.Run("nil config", func(t *testing.T) { - _, err := parseFeishuConfig(nil, RegionFeishu) + _, err := core.ParseFeishuConfig(nil, core.RegionFeishu) if err == nil { t.Fatal("expected error for nil config") } }) t.Run("missing credentials", func(t *testing.T) { - _, err := parseFeishuConfig(&types.DataSourceConfig{ + _, err := core.ParseFeishuConfig(&types.DataSourceConfig{ Credentials: map[string]interface{}{ "app_id": "id1", // missing app_secret }, - }, RegionFeishu) + }, core.RegionFeishu) if err == nil { t.Fatal("expected error for missing app_secret") } @@ -438,7 +391,7 @@ func TestParseFeishuConfig(t *testing.T) { // ────────────────────────────────────────────────────────────────────── func TestConnectorType(t *testing.T) { - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) if c.Type() != types.ConnectorTypeFeishu { t.Errorf("Type() = %q, want %q", c.Type(), types.ConnectorTypeFeishu) } @@ -448,7 +401,7 @@ func TestConnectorValidate(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) err := c.Validate(context.Background(), makeConfig(cfg, nil)) if err != nil { t.Fatalf("Validate() error: %v", err) @@ -456,7 +409,7 @@ func TestConnectorValidate(t *testing.T) { } func TestConnectorValidate_BadCredentials(t *testing.T) { - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) err := c.Validate(context.Background(), &types.DataSourceConfig{ Credentials: map[string]interface{}{ "app_id": "bad", @@ -473,7 +426,7 @@ func TestConnectorListResources(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) resources, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "") if err != nil { t.Fatalf("ListResources() error: %v", err) @@ -500,11 +453,11 @@ func TestConnectorListResources(t *testing.T) { // the wiki tree lazily — only the requested level — instead of recursing the whole // tree up front (Tencent/WeKnora#1672). func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) { - topNodes := []wikiNode{ + topNodes := []core.WikiNode{ {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true, ObjEditTime: "100"}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "docx", Title: "Peer", ObjEditTime: "200"}, } - childNodes := map[string][]wikiNode{ + childNodes := map[string][]core.WikiNode{ "nt-root": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "docx", Title: "Child", ObjEditTime: "300"}, }, @@ -512,7 +465,7 @@ func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) { ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "") defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) // Root listing: only the space, no descendants. spaces, err := c.ListResources(context.Background(), makeConfig(cfg, nil), "") @@ -561,10 +514,10 @@ func TestConnectorListResources_LazyLoadsOneLevel(t *testing.T) { // deeply nested selection is resolved (so an edit-mode picker can reveal it) // without listing the whole tree. func TestConnectorResolveResourceAncestors(t *testing.T) { - topNodes := []wikiNode{ + topNodes := []core.WikiNode{ {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "docx", Title: "Root", HasChild: true}, } - childNodes := map[string][]wikiNode{ + childNodes := map[string][]core.WikiNode{ "nt-root": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "docx", Title: "Child", HasChild: true}, }, @@ -575,7 +528,7 @@ func TestConnectorResolveResourceAncestors(t *testing.T) { ts, cfg := fakeFeishuHierarchy(topNodes, childNodes, "") defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) // A deeply nested node resolves to its space plus every intermediate parent. ancestors, err := c.ResolveResourceAncestors( @@ -625,7 +578,7 @@ func TestConnectorResolveResourceAncestors(t *testing.T) { // ────────────────────────────────────────────────────────────────────── func TestFetchAll_DocxNode(t *testing.T) { - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: "nt1", ObjToken: "obj-docx-1", ObjType: "docx", @@ -635,7 +588,7 @@ func TestFetchAll_DocxNode(t *testing.T) { ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -667,7 +620,7 @@ func TestFetchAll_DocxNode(t *testing.T) { } func TestFetchAll_SheetNode(t *testing.T) { - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: "nt-sheet", ObjToken: "obj-sheet-1", ObjType: "sheet", @@ -677,7 +630,7 @@ func TestFetchAll_SheetNode(t *testing.T) { ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -692,7 +645,7 @@ func TestFetchAll_SheetNode(t *testing.T) { } func TestFetchAll_BitableNode(t *testing.T) { - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: "nt-bitable", ObjToken: "obj-bitable-1", ObjType: "bitable", @@ -702,7 +655,7 @@ func TestFetchAll_BitableNode(t *testing.T) { ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -716,7 +669,7 @@ func TestFetchAll_BitableNode(t *testing.T) { } func TestFetchAll_FileNode(t *testing.T) { - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: "nt-file", ObjToken: "obj-file-1", ObjType: "file", @@ -726,7 +679,7 @@ func TestFetchAll_FileNode(t *testing.T) { ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -749,27 +702,27 @@ func TestFetchAll_FileNode(t *testing.T) { } func TestFetchAll_SkipsMindnoteAndSlides(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt-mn", ObjToken: "obj-mn", ObjType: "mindnote", Title: "Brain Map"}, {NodeToken: "nt-sl", ObjToken: "obj-sl", ObjType: "slides", Title: "Presentation"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } - // Both should be skipped (nil returned by fetchNodeContent) + // Both should be core.Skipped (nil returned by fetchNodeContent) if len(items) != 0 { - t.Errorf("expected 0 items (mindnote+slides skipped), got %d", len(items)) + t.Errorf("expected 0 items (mindnote+slides core.Skipped), got %d", len(items)) } } func TestFetchAll_MixedTypes(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "sheet", Title: "Sheet", NodeEditTime: "1711468800"}, {NodeToken: "nt3", ObjToken: "obj3", ObjType: "file", Title: "report.pdf", NodeEditTime: "1711468800"}, @@ -780,13 +733,13 @@ func TestFetchAll_MixedTypes(t *testing.T) { ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) items, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) } - // docx + sheet + file + bitable = 4 items; mindnote + slides = skipped + // docx + sheet + file + bitable = 4 items; mindnote + slides = core.Skipped if len(items) != 4 { t.Errorf("expected 4 items, got %d", len(items)) for i, it := range items { @@ -796,7 +749,7 @@ func TestFetchAll_MixedTypes(t *testing.T) { } func TestFetchAll_LogsSummaryWithSkipBreakdown(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc", NodeEditTime: "1711468800"}, {NodeToken: "nt4", ObjToken: "obj4", ObjType: "mindnote", Title: "Mind", NodeEditTime: "1711468800"}, {NodeToken: "nt5", ObjToken: "obj5", ObjType: "slides", Title: "Slides", NodeEditTime: "1711468800"}, @@ -808,14 +761,14 @@ func TestFetchAll_LogsSummaryWithSkipBreakdown(t *testing.T) { logger.SetOutput(&buf) defer logger.SetOutput(os.Stderr) - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) if _, err := c.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}); err != nil { t.Fatalf("FetchAll() error: %v", err) } out := buf.String() for _, want := range []string{ - "sync summary", + "stream summary", "discovered=3", "fetched=1", "skipped_unsupported=2", @@ -829,14 +782,14 @@ func TestFetchAll_LogsSummaryWithSkipBreakdown(t *testing.T) { } func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, } ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent") defer ts.Close() - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll must not abort when one child listing fails: %v", err) @@ -879,11 +832,11 @@ func TestFetchAll_ChildNodeListErrorReturnsPartialItems(t *testing.T) { } func TestFetchAll_WikiNodeResourceSyncsSelectedSubtree(t *testing.T) { - topNodes := []wikiNode{ + topNodes := []core.WikiNode{ {NodeToken: "nt-root", ObjToken: "obj-root", ObjType: "file", Title: "Root.pdf", NodeEditTime: "100", HasChild: true}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, } - childNodes := map[string][]wikiNode{ + childNodes := map[string][]core.WikiNode{ "nt-root": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "file", Title: "Child.pdf", NodeEditTime: "300"}, }, @@ -892,7 +845,7 @@ func TestFetchAll_WikiNodeResourceSyncsSelectedSubtree(t *testing.T) { defer ts.Close() resourceID := "space1:nt-root" - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{resourceID}), []string{resourceID}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -924,14 +877,14 @@ func TestFetchAll_WikiNodeResourceSyncsSelectedSubtree(t *testing.T) { // ────────────────────────────────────────────────────────────────────── func TestFetchIncremental_FirstSync(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "file", Title: "file.pdf", NodeEditTime: "200"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, []string{"space1"}) // First sync with no cursor → all items should be fetched @@ -952,13 +905,13 @@ func TestFetchIncremental_FirstSync(t *testing.T) { } func TestFetchIncremental_NoChanges(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, } ts, cfg := fakeFeishu(nodes) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, []string{"space1"}) // First sync @@ -980,13 +933,13 @@ func TestFetchIncremental_NoChanges(t *testing.T) { func TestFetchIncremental_DetectsDeleted(t *testing.T) { // First sync: 2 nodes - allNodes := []wikiNode{ + allNodes := []core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, {NodeToken: "nt2", ObjToken: "obj2", ObjType: "docx", Title: "Doc2", NodeEditTime: "200"}, } ts, cfg := fakeFeishu(allNodes) - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, []string{"space1"}) _, cursor1, err := c.FetchIncremental(context.Background(), dsConfig, nil) @@ -996,7 +949,7 @@ func TestFetchIncremental_DetectsDeleted(t *testing.T) { ts.Close() // Second sync: only 1 node remains (nt2 was deleted) - ts2, cfg2 := fakeFeishu([]wikiNode{ + ts2, cfg2 := fakeFeishu([]core.WikiNode{ {NodeToken: "nt1", ObjToken: "obj1", ObjType: "docx", Title: "Doc1", NodeEditTime: "100"}, }) defer ts2.Close() @@ -1026,7 +979,7 @@ func TestFetchIncremental_NoResourceIDs(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - c := NewConnector(RegionFeishu) + c := NewConnector(core.RegionFeishu) dsConfig := makeConfig(cfg, nil) // no resource IDs dsConfig.ResourceIDs = nil @@ -1040,14 +993,14 @@ func TestFetchIncremental_NoResourceIDs(t *testing.T) { } func TestFetchIncremental_ChildNodeListErrorReturnsPartialItemsAndCursor(t *testing.T) { - nodes := []wikiNode{ + nodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, {NodeToken: "nt-peer", ObjToken: "obj-peer", ObjType: "file", Title: "Peer.pdf", NodeEditTime: "200"}, } ts, cfg := fakeFeishuWithChildFailure(nodes, "nt-parent") defer ts.Close() - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) items, cursor, err := conn.FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil) if err != nil { t.Fatalf("FetchIncremental must not abort when one child listing fails: %v", err) @@ -1075,17 +1028,17 @@ func TestFetchIncremental_ChildNodeListErrorReturnsPartialItemsAndCursor(t *test } func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren(t *testing.T) { - firstNodes := []wikiNode{ + firstNodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, } - firstChildren := map[string][]wikiNode{ + firstChildren := map[string][]core.WikiNode{ "nt-parent": { {NodeToken: "nt-child", ObjToken: "obj-child", ObjType: "file", Title: "Child.pdf", NodeEditTime: "150"}, }, } ts, cfg := fakeFeishuHierarchy(firstNodes, firstChildren, "") - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) firstItems, cursor, err := conn.FetchIncremental(context.Background(), makeConfig(cfg, []string{"space1"}), nil) if err != nil { t.Fatalf("first sync error: %v", err) @@ -1095,7 +1048,7 @@ func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren( } ts.Close() - secondNodes := []wikiNode{ + secondNodes := []core.WikiNode{ {NodeToken: "nt-parent", ObjToken: "obj-parent", ObjType: "file", Title: "Parent.pdf", NodeEditTime: "100", HasChild: true}, } ts2, cfg2 := fakeFeishuHierarchy(secondNodes, nil, "nt-parent") @@ -1112,7 +1065,7 @@ func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren( } cursorBytes, _ := json.Marshal(nextCursor.ConnectorCursor) - var restored feishuCursor + var restored core.FeishuCursor if err := json.Unmarshal(cursorBytes, &restored); err != nil { t.Fatalf("restore cursor: %v", err) } @@ -1122,16 +1075,16 @@ func TestFetchIncremental_ChildNodeListErrorDoesNotDeletePreviouslySeenChildren( } // ────────────────────────────────────────────────────────────────────── -// Client tests +// core.Client tests // ────────────────────────────────────────────────────────────────────── func TestClientPing(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - client := NewClient(cfg) + client := core.NewClient(cfg) if err := client.Ping(context.Background()); err != nil { - t.Fatalf("Ping() error: %v", err) + t.Fatalf("core.Ping() error: %v", err) } } @@ -1140,8 +1093,8 @@ func TestClientTokenCaching(t *testing.T) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { callCount++ - writeJSON(w, tokenResponse{ - apiResponse: apiResponse{Code: 0}, + writeJSON(w, core.TokenResponse{ + ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: fmt.Sprintf("token-%d", callCount), Expire: 7200, }) @@ -1149,12 +1102,12 @@ func TestClientTokenCaching(t *testing.T) { ts := httptest.NewServer(mux) defer ts.Close() - client := NewClient(&Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}) + client := core.NewClient(&core.Config{AppID: "a", AppSecret: "b", BaseURL: ts.URL}) // First call: fetches token - t1, _ := client.getTenantAccessToken(context.Background()) + t1, _ := client.GetTenantAccessToken(context.Background()) // Second call: should use cache - t2, _ := client.getTenantAccessToken(context.Background()) + t2, _ := client.GetTenantAccessToken(context.Background()) if t1 != t2 { t.Errorf("expected cached token, got different tokens: %q vs %q", t1, t2) @@ -1168,10 +1121,10 @@ func TestClientExportAndDownload(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - client := NewClient(cfg) + client := core.NewClient(cfg) data, fileName, err := client.ExportAndDownload(context.Background(), "obj-token-1", "docx") if err != nil { - t.Fatalf("ExportAndDownload() error: %v", err) + t.Fatalf("core.ExportAndDownload() error: %v", err) } if string(data) != "fake-docx-content" { @@ -1186,7 +1139,7 @@ func TestClientExportAndDownload_UnsupportedType(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - client := NewClient(cfg) + client := core.NewClient(cfg) _, _, err := client.ExportAndDownload(context.Background(), "obj-token-1", "mindnote") if err == nil { t.Fatal("expected error for unsupported type") @@ -1200,10 +1153,10 @@ func TestClientDownloadDriveFile(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - client := NewClient(cfg) + client := core.NewClient(cfg) data, err := client.DownloadDriveFile(context.Background(), "file-token-abc") if err != nil { - t.Fatalf("DownloadDriveFile() error: %v", err) + t.Fatalf("core.DownloadDriveFile() error: %v", err) } if string(data) != "fake-pdf-binary" { @@ -1215,10 +1168,10 @@ func TestClientListWikiSpaces(t *testing.T) { ts, cfg := fakeFeishu(nil) defer ts.Close() - client := NewClient(cfg) + client := core.NewClient(cfg) spaces, err := client.ListWikiSpaces(context.Background()) if err != nil { - t.Fatalf("ListWikiSpaces() error: %v", err) + t.Fatalf("core.ListWikiSpaces() error: %v", err) } if len(spaces) != 1 { t.Fatalf("expected 1 space, got %d", len(spaces)) @@ -1236,32 +1189,32 @@ func TestObjTypeToExportMappings(t *testing.T) { // Verify all exportable types have valid mappings exportable := []string{"docx", "doc", "sheet", "bitable"} for _, ot := range exportable { - if _, ok := objTypeToExportFileExtension[ot]; !ok { - t.Errorf("objTypeToExportFileExtension missing %q", ot) + if _, ok := core.ObjTypeToExportFileExtension[ot]; !ok { + t.Errorf("core.ObjTypeToExportFileExtension missing %q", ot) } - if _, ok := objTypeToExportType[ot]; !ok { - t.Errorf("objTypeToExportType missing %q", ot) + if _, ok := core.ObjTypeToExportType[ot]; !ok { + t.Errorf("core.ObjTypeToExportType missing %q", ot) } } // Verify non-exportable types do NOT have mappings nonExportable := []string{"file", "mindnote", "slides"} for _, ot := range nonExportable { - if _, ok := objTypeToExportFileExtension[ot]; ok { - t.Errorf("objTypeToExportFileExtension should NOT contain %q", ot) + if _, ok := core.ObjTypeToExportFileExtension[ot]; ok { + t.Errorf("core.ObjTypeToExportFileExtension should NOT contain %q", ot) } } } func TestExportFileExtToSuffix(t *testing.T) { - if exportFileExtToSuffix[ExportTypeDocx] != ".docx" { - t.Errorf("docx suffix = %q", exportFileExtToSuffix[ExportTypeDocx]) + if core.ExportFileExtToSuffix[core.ExportTypeDocx] != ".docx" { + t.Errorf("docx suffix = %q", core.ExportFileExtToSuffix[core.ExportTypeDocx]) } - if exportFileExtToSuffix[ExportTypeXlsx] != ".xlsx" { - t.Errorf("xlsx suffix = %q", exportFileExtToSuffix[ExportTypeXlsx]) + if core.ExportFileExtToSuffix[core.ExportTypeXlsx] != ".xlsx" { + t.Errorf("xlsx suffix = %q", core.ExportFileExtToSuffix[core.ExportTypeXlsx]) } - if exportFileExtToSuffix[ExportTypePDF] != ".pdf" { - t.Errorf("pdf suffix = %q", exportFileExtToSuffix[ExportTypePDF]) + if core.ExportFileExtToSuffix[core.ExportTypePDF] != ".pdf" { + t.Errorf("pdf suffix = %q", core.ExportFileExtToSuffix[core.ExportTypePDF]) } } @@ -1277,58 +1230,41 @@ func TestExportFileExtToSuffix(t *testing.T) { // // The export endpoint is intentionally absent; any call to it returns 404 so the // test verifies the blocks-API path, not the export-fallback path. -func fakeFeishuWithBlocks(nodes []wikiNode, docToken, attToken, attName string, attContent []byte) (*httptest.Server, *Config) { +func fakeFeishuWithBlocks(nodes []core.WikiNode, docToken, attToken, attName string, attContent []byte) (*httptest.Server, *core.Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{ - apiResponse: apiResponse{Code: 0}, + writeJSON(w, core.TokenResponse{ + ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, + writeJSON(w, core.WikiSpaceListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiNodeListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: nodes}, + writeJSON(w, core.WikiNodeListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeListData{Items: nodes}, }) }) // blocks API for the given docx document blocksPath := "/open-apis/docx/v1/documents/" + docToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, docxBlocksResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []docxBlock `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ - Items: []docxBlock{ - {BlockID: "b1", BlockType: blockTypePage}, - {BlockID: "b2", BlockType: blockTypeText, Text: &blockText{ - Elements: []textElement{{TextRun: &struct { - Content string `json:"content"` - }{Content: "Hello blocks"}}}, + writeJSON(w, core.DocxBlocksResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.DocxBlocksData{ + Items: []core.DocxBlock{ + {BlockID: "b1", BlockType: core.BlockTypePage}, + {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ + Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "Hello blocks"}}}, }}, - {BlockID: "b3", BlockType: blockTypeFile, File: &struct { - Token string `json:"token"` - Name string `json:"name"` - }{Token: attToken, Name: attName}}, + {BlockID: "b3", BlockType: core.BlockTypeFile, File: &core.BlockFileRef{Token: attToken, Name: attName}}, }, }, }) @@ -1345,22 +1281,23 @@ func fakeFeishuWithBlocks(nodes []wikiNode, docToken, attToken, attName string, }) ts := httptest.NewServer(mux) - return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} + return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} } // TestFetchDocxWithBlocks_MultiItem verifies that a docx node returns a main // Markdown item plus an attachment sub-item when the blocks API succeeds. func TestFetchDocxWithBlocks_MultiItem(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx" objToken = "obj-docx" attToken = "ft-att-1" attName = "report.pdf" ) - // Attachment content must exceed minAttachmentBytes (2 KiB) to not be filtered. - attContent := bytes.Repeat([]byte("x"), minAttachmentBytes+1) + // Attachment content must exceed core.MinAttachmentBytes (2 KiB) to not be filtered. + attContent := bytes.Repeat([]byte("x"), core.MinAttachmentBytes+1) - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", @@ -1370,7 +1307,7 @@ func TestFetchDocxWithBlocks_MultiItem(t *testing.T) { ts, cfg := fakeFeishuWithBlocks(nodes, objToken, attToken, attName, attContent) defer ts.Close() - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -1411,58 +1348,41 @@ func TestFetchDocxWithBlocks_MultiItem(t *testing.T) { // drive download endpoint returns the given HTTP status code instead of 200. Use // downloadStatus = http.StatusInternalServerError to exercise the // attachment-download-failure path. -func fakeFeishuWithBlocksAndDownloadStatus(nodes []wikiNode, docToken, attToken, attName string, downloadStatus int) (*httptest.Server, *Config) { +func fakeFeishuWithBlocksAndDownloadStatus(nodes []core.WikiNode, docToken, attToken, attName string, downloadStatus int) (*httptest.Server, *core.Config) { mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{ - apiResponse: apiResponse{Code: 0}, + writeJSON(w, core.TokenResponse{ + ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiSpaceListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiSpace `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: []wikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, + writeJSON(w, core.WikiSpaceListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiSpaceListData{Items: []core.WikiSpace{{SpaceID: "space1", Name: "Test Space"}}}, }) }) mux.HandleFunc("/open-apis/wiki/v2/spaces/space1/nodes", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, wikiNodeListResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []wikiNode `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{Items: nodes}, + writeJSON(w, core.WikiNodeListResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.WikiNodeListData{Items: nodes}, }) }) // blocks API — one text block + one parseable .pdf file block blocksPath := "/open-apis/docx/v1/documents/" + docToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, docxBlocksResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []docxBlock `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ - Items: []docxBlock{ - {BlockID: "b1", BlockType: blockTypePage}, - {BlockID: "b2", BlockType: blockTypeText, Text: &blockText{ - Elements: []textElement{{TextRun: &struct { - Content string `json:"content"` - }{Content: "Hello"}}}, + writeJSON(w, core.DocxBlocksResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.DocxBlocksData{ + Items: []core.DocxBlock{ + {BlockID: "b1", BlockType: core.BlockTypePage}, + {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ + Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "Hello"}}}, }}, - {BlockID: "b3", BlockType: blockTypeFile, File: &struct { - Token string `json:"token"` - Name string `json:"name"` - }{Token: attToken, Name: attName}}, + {BlockID: "b3", BlockType: core.BlockTypeFile, File: &core.BlockFileRef{Token: attToken, Name: attName}}, }, }, }) @@ -1479,7 +1399,7 @@ func fakeFeishuWithBlocksAndDownloadStatus(nodes []wikiNode, docToken, attToken, }) ts := httptest.NewServer(mux) - return ts, &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} + return ts, &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} } // TestFetchDocxWithBlocks_AttachmentDownloadFailure verifies that when the blocks @@ -1489,13 +1409,14 @@ func fakeFeishuWithBlocksAndDownloadStatus(nodes []wikiNode, docToken, attToken, // subtree sweep is suppressed so a transient failure never deletes the good prior // copy of the attachment. One bad attachment must not block the whole document. func TestFetchDocxWithBlocks_AttachmentDownloadFailure(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-fail" objToken = "obj-docx-fail" attToken = "ft-att-bad" attName = "slides.pdf" ) - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", @@ -1505,9 +1426,9 @@ func TestFetchDocxWithBlocks_AttachmentDownloadFailure(t *testing.T) { ts, cfg := fakeFeishuWithBlocksAndDownloadStatus(nodes, objToken, attToken, attName, http.StatusInternalServerError) defer ts.Close() - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) ctx := context.Background() - client := NewClient(cfg) + client := core.NewClient(cfg) baseMeta := map[string]string{ "obj_token": objToken, @@ -1516,7 +1437,16 @@ func TestFetchDocxWithBlocks_AttachmentDownloadFailure(t *testing.T) { "space_id": "space1", "channel": types.ChannelFeishu, } - items, err := conn.fetchDocxWithBlocks(ctx, client, nodes[0], "space1:nt-docx-fail", parseFeishuTimestamp("1711468800"), baseMeta, true) + items, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ + DocToken: nodeToken, + ObjToken: objToken, + Title: "Doc With Bad Attachment", + URL: conn.region.WikiURL(nodeToken), + ResourceID: "space1:nt-docx-fail", + EditTime: core.ParseFeishuTimestamp("1711468800"), + BaseMeta: baseMeta, + MultimodalEnabled: true, + }) if err != nil { t.Fatalf("a failed attachment must not fail the whole node, got error: %v", err) } @@ -1565,38 +1495,31 @@ func TestFetchDocxWithBlocks_AttachmentDownloadFailure(t *testing.T) { // the KB has multimodal enabled, but their external_id is ALWAYS kept in // SubtreeKeep so toggling VLM off later does not sweep previously OCR'd images. func TestFetchDocxWithBlocks_EmbeddedImage(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-img" objToken = "obj-docx-img" imgToken = "media-img-1" ) // A valid PNG signature + padding so http.DetectContentType returns image/png - // and the bytes exceed minAttachmentBytes. - pngBytes := append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte("x"), minAttachmentBytes)...) + // and the bytes exceed core.MinAttachmentBytes. + pngBytes := append([]byte("\x89PNG\r\n\x1a\n"), bytes.Repeat([]byte("x"), core.MinAttachmentBytes)...) mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) + writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) }) blocksPath := "/open-apis/docx/v1/documents/" + objToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, docxBlocksResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []docxBlock `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ - Items: []docxBlock{ - {BlockID: "b1", BlockType: blockTypePage}, - {BlockID: "b2", BlockType: blockTypeText, Text: &blockText{ - Elements: []textElement{{TextRun: &struct { - Content string `json:"content"` - }{Content: "Hello"}}}, + writeJSON(w, core.DocxBlocksResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.DocxBlocksData{ + Items: []core.DocxBlock{ + {BlockID: "b1", BlockType: core.BlockTypePage}, + {BlockID: "b2", BlockType: core.BlockTypeText, Text: &core.BlockText{ + Elements: []core.TextElement{{TextRun: &core.TextRun{Content: "Hello"}}}, }}, - {BlockID: "b3", BlockType: blockTypeImage, Image: &struct { - Token string `json:"token"` - }{Token: imgToken}}, + {BlockID: "b3", BlockType: core.BlockTypeImage, Image: &core.BlockTokenRef{Token: imgToken}}, }, }, }) @@ -1611,18 +1534,27 @@ func TestFetchDocxWithBlocks_EmbeddedImage(t *testing.T) { ts := httptest.NewServer(mux) defer ts.Close() - cfg := &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} - conn := NewConnector(RegionFeishu) + cfg := &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} + conn := NewConnector(core.RegionFeishu) ctx := context.Background() - client := NewClient(cfg) - node := wikiNode{NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Image", NodeEditTime: "1711468800"} + client := core.NewClient(cfg) + node := core.WikiNode{NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Image", NodeEditTime: "1711468800"} baseMeta := map[string]string{"node_token": nodeToken, "channel": types.ChannelFeishu} imgChildID := nodeToken + "#image#" + imgToken // multimodal ON → image emitted as a sub-item and kept. - items, err := conn.fetchDocxWithBlocks(ctx, client, node, "space1:"+nodeToken, parseFeishuTimestamp("1711468800"), baseMeta, true) + items, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ + DocToken: node.NodeToken, + ObjToken: node.ObjToken, + Title: node.Title, + URL: conn.region.WikiURL(node.NodeToken), + ResourceID: "space1:" + nodeToken, + EditTime: core.ParseFeishuTimestamp("1711468800"), + BaseMeta: baseMeta, + MultimodalEnabled: true, + }) if err != nil { - t.Fatalf("fetchDocxWithBlocks (multimodal on): %v", err) + t.Fatalf("core.FetchDocxWithBlocks (multimodal on): %v", err) } var main, img *types.FetchedItem for _, it := range items { @@ -1656,14 +1588,23 @@ func TestFetchDocxWithBlocks_EmbeddedImage(t *testing.T) { } // multimodal OFF → no image sub-item, but the id is still kept (not swept). - itemsOff, err := conn.fetchDocxWithBlocks(ctx, client, node, "space1:"+nodeToken, parseFeishuTimestamp("1711468800"), baseMeta, false) + itemsOff, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ + DocToken: node.NodeToken, + ObjToken: node.ObjToken, + Title: node.Title, + URL: conn.region.WikiURL(node.NodeToken), + ResourceID: "space1:" + nodeToken, + EditTime: core.ParseFeishuTimestamp("1711468800"), + BaseMeta: baseMeta, + MultimodalEnabled: false, + }) if err != nil { - t.Fatalf("fetchDocxWithBlocks (multimodal off): %v", err) + t.Fatalf("core.FetchDocxWithBlocks (multimodal off): %v", err) } var mainOff *types.FetchedItem for _, it := range itemsOff { if it.ExternalID == imgChildID { - t.Errorf("multimodal off must NOT emit an image sub-item, got %+v", it) + t.Errorf("multimodal off must NOT Emit an image sub-item, got %+v", it) } if it.ExternalID == nodeToken { mainOff = it @@ -1679,11 +1620,12 @@ func TestFetchDocxWithBlocks_EmbeddedImage(t *testing.T) { } // TestFetchDocxWithBlocks_ImageDownloadFailure verifies that a failed image -// download (a genuine fetch failure — revoked token, permission gap, transient +// download (a genuine core.Fetch failure — revoked token, permission gap, transient // error) surfaces a visible error sub-item exactly like a failed attachment // download, rather than being silently dropped to a server log. The image is // still kept in SubtreeKeep so any prior OCR'd copy is preserved. func TestFetchDocxWithBlocks_ImageDownloadFailure(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-img-fail" objToken = "obj-docx-img-fail" @@ -1691,22 +1633,16 @@ func TestFetchDocxWithBlocks_ImageDownloadFailure(t *testing.T) { ) mux := http.NewServeMux() mux.HandleFunc("/open-apis/auth/v3/tenant_access_token/internal", func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, tokenResponse{apiResponse: apiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) + writeJSON(w, core.TokenResponse{ApiResponse: core.ApiResponse{Code: 0}, TenantAccessToken: "fake-token", Expire: 7200}) }) blocksPath := "/open-apis/docx/v1/documents/" + objToken + "/blocks" mux.HandleFunc(blocksPath, func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, docxBlocksResponse{ - apiResponse: apiResponse{Code: 0}, - Data: struct { - Items []docxBlock `json:"items"` - HasMore bool `json:"has_more"` - PageToken string `json:"page_token"` - }{ - Items: []docxBlock{ - {BlockID: "b1", BlockType: blockTypePage}, - {BlockID: "b2", BlockType: blockTypeImage, Image: &struct { - Token string `json:"token"` - }{Token: imgToken}}, + writeJSON(w, core.DocxBlocksResponse{ + ApiResponse: core.ApiResponse{Code: 0}, + Data: core.DocxBlocksData{ + Items: []core.DocxBlock{ + {BlockID: "b1", BlockType: core.BlockTypePage}, + {BlockID: "b2", BlockType: core.BlockTypeImage, Image: &core.BlockTokenRef{Token: imgToken}}, }, }, }) @@ -1718,16 +1654,25 @@ func TestFetchDocxWithBlocks_ImageDownloadFailure(t *testing.T) { ts := httptest.NewServer(mux) defer ts.Close() - cfg := &Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} - conn := NewConnector(RegionFeishu) + cfg := &core.Config{AppID: "test-app-id", AppSecret: "test-app-secret", BaseURL: ts.URL} + conn := NewConnector(core.RegionFeishu) ctx := context.Background() - client := NewClient(cfg) - node := wikiNode{NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Bad Image", NodeEditTime: "1711468800"} + client := core.NewClient(cfg) + node := core.WikiNode{NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", Title: "Doc With Bad Image", NodeEditTime: "1711468800"} baseMeta := map[string]string{"node_token": nodeToken, "channel": types.ChannelFeishu} imgChildID := nodeToken + "#image#" + imgToken // multimodal ON → the download is attempted and fails → a visible error item. - items, err := conn.fetchDocxWithBlocks(ctx, client, node, "space1:"+nodeToken, parseFeishuTimestamp("1711468800"), baseMeta, true) + items, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ + DocToken: node.NodeToken, + ObjToken: node.ObjToken, + Title: node.Title, + URL: conn.region.WikiURL(node.NodeToken), + ResourceID: "space1:" + nodeToken, + EditTime: core.ParseFeishuTimestamp("1711468800"), + BaseMeta: baseMeta, + MultimodalEnabled: true, + }) if err != nil { t.Fatalf("a failed image download must not fail the whole node, got error: %v", err) } @@ -1744,7 +1689,7 @@ func TestFetchDocxWithBlocks_ImageDownloadFailure(t *testing.T) { t.Fatal("main doc item missing") } if errItem == nil { - t.Fatalf("failed image download must emit a visible error sub-item, got %+v", items) + t.Fatalf("failed image download must Emit a visible error sub-item, got %+v", items) } if len(errItem.Content) != 0 { t.Errorf("image error item must carry no content, got %d bytes", len(errItem.Content)) @@ -1762,13 +1707,22 @@ func TestFetchDocxWithBlocks_ImageDownloadFailure(t *testing.T) { // multimodal OFF → the download is never attempted, so no error item, but the // id is still kept (not swept). - itemsOff, err := conn.fetchDocxWithBlocks(ctx, client, node, "space1:"+nodeToken, parseFeishuTimestamp("1711468800"), baseMeta, false) + itemsOff, err := core.FetchDocxWithBlocks(ctx, client, core.DocxFetchInput{ + DocToken: node.NodeToken, + ObjToken: node.ObjToken, + Title: node.Title, + URL: conn.region.WikiURL(node.NodeToken), + ResourceID: "space1:" + nodeToken, + EditTime: core.ParseFeishuTimestamp("1711468800"), + BaseMeta: baseMeta, + MultimodalEnabled: false, + }) if err != nil { - t.Fatalf("fetchDocxWithBlocks (multimodal off): %v", err) + t.Fatalf("core.FetchDocxWithBlocks (multimodal off): %v", err) } for _, it := range itemsOff { if it.ExternalID == imgChildID { - t.Errorf("multimodal off must NOT attempt the image download or emit an item, got %+v", it) + t.Errorf("multimodal off must NOT attempt the image download or Emit an item, got %+v", it) } } } @@ -1777,16 +1731,17 @@ func TestFetchDocxWithBlocks_ImageDownloadFailure(t *testing.T) { // whose extension is not in the parseable-attachment whitelist (e.g. .png) is NOT // promoted to a sub-item, but its inline reference IS present in the main document. func TestFetchDocxWithBlocks_NonWhitelistedExtNotPromoted(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-png" objToken = "obj-docx-png" attToken = "ft-icon" attName = "icon.png" ) - // Content size well above minAttachmentBytes — the filter must be extension, not size. - attContent := bytes.Repeat([]byte("x"), minAttachmentBytes+100) + // Content size well above core.MinAttachmentBytes — the filter must be extension, not size. + attContent := bytes.Repeat([]byte("x"), core.MinAttachmentBytes+100) - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", @@ -1796,7 +1751,7 @@ func TestFetchDocxWithBlocks_NonWhitelistedExtNotPromoted(t *testing.T) { ts, cfg := fakeFeishuWithBlocks(nodes, objToken, attToken, attName, attContent) defer ts.Close() - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -1813,19 +1768,20 @@ func TestFetchDocxWithBlocks_NonWhitelistedExtNotPromoted(t *testing.T) { } // TestFetchDocxWithBlocks_WhitelistedTinyAttachmentNotPromoted verifies that a -// whitelisted-extension file block whose download is smaller than minAttachmentBytes +// whitelisted-extension file block whose download is smaller than core.MinAttachmentBytes // is NOT promoted to a sub-item, but its inline reference IS present in the main doc. func TestFetchDocxWithBlocks_WhitelistedTinyAttachmentNotPromoted(t *testing.T) { + t.Setenv("FEISHU_DOCX_PARSE_MODE", "blocks") const ( nodeToken = "nt-docx-tiny" objToken = "obj-docx-tiny" attToken = "ft-tiny-pdf" attName = "tiny.pdf" ) - // Content is well below minAttachmentBytes (100 bytes vs 2048 threshold). + // Content is well below core.MinAttachmentBytes (100 bytes vs 2048 threshold). attContent := bytes.Repeat([]byte("x"), 100) - nodes := []wikiNode{{ + nodes := []core.WikiNode{{ NodeToken: nodeToken, ObjToken: objToken, ObjType: "docx", @@ -1835,7 +1791,7 @@ func TestFetchDocxWithBlocks_WhitelistedTinyAttachmentNotPromoted(t *testing.T) ts, cfg := fakeFeishuWithBlocks(nodes, objToken, attToken, attName, attContent) defer ts.Close() - conn := NewConnector(RegionFeishu) + conn := NewConnector(core.RegionFeishu) items, err := conn.FetchAll(context.Background(), makeConfig(cfg, []string{"space1"}), []string{"space1"}) if err != nil { t.Fatalf("FetchAll() error: %v", err) @@ -1856,7 +1812,7 @@ func TestFetchDocxWithBlocks_WhitelistedTinyAttachmentNotPromoted(t *testing.T) // ────────────────────────────────────────────────────────────────────── func TestFeishuCursorRoundTrip(t *testing.T) { - original := feishuCursor{ + original := core.FeishuCursor{ LastSyncTime: time.Date(2026, 3, 26, 12, 0, 0, 0, time.UTC), SpaceNodeTimes: map[string]map[string]string{ "space1": { @@ -1879,7 +1835,7 @@ func TestFeishuCursorRoundTrip(t *testing.T) { // Deserialize back data2, _ := json.Marshal(cursorMap) - var restored feishuCursor + var restored core.FeishuCursor if err := json.Unmarshal(data2, &restored); err != nil { t.Fatalf("restore error: %v", err) } @@ -1912,7 +1868,7 @@ func TestSupportedImageExt(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - ext, ct, ok := supportedImageExt(tc.data) + ext, ct, ok := core.SupportedImageExt(tc.data) if ok != tc.wantOK { t.Fatalf("ok = %v, want %v", ok, tc.wantOK) } diff --git a/internal/datasource/connector/feishu/wiki/helpers_test.go b/internal/datasource/connector/feishu/wiki/helpers_test.go new file mode 100644 index 000000000..8f76d4d8f --- /dev/null +++ b/internal/datasource/connector/feishu/wiki/helpers_test.go @@ -0,0 +1,7 @@ +package wiki + +import "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" + +func txt(s string) *core.BlockText { + return &core.BlockText{Elements: []core.TextElement{{TextRun: &core.TextRun{Content: s}}}} +} diff --git a/internal/datasource/connector/feishu/lark_e2e_test.go b/internal/datasource/connector/feishu/wiki/lark_e2e_test.go similarity index 76% rename from internal/datasource/connector/feishu/lark_e2e_test.go rename to internal/datasource/connector/feishu/wiki/lark_e2e_test.go index a3b25f5dd..7382ce8b6 100644 --- a/internal/datasource/connector/feishu/lark_e2e_test.go +++ b/internal/datasource/connector/feishu/wiki/lark_e2e_test.go @@ -1,10 +1,11 @@ -package feishu +package wiki import ( "context" "strings" "testing" + "github.com/Tencent/WeKnora/internal/datasource/connector/feishu/core" "github.com/Tencent/WeKnora/internal/types" ) @@ -14,15 +15,15 @@ import ( // // The fake server stands in for the Open Platform (base_url points at it), but // the resource URL is built from the region's web host, so this exercises the -// real ListResources → wikiNodeToResource → region.wikiURL path. +// real ListResources → wikiNodeToResource → region.WikiURL path. func TestListResources_ResourceURLFollowsRegion(t *testing.T) { cases := []struct { - region Region + region core.Region wantURL string forbiddenIn string }{ - {RegionFeishu, "https://feishu.cn/wiki/space1", "larksuite"}, - {RegionLark, "https://larksuite.com/wiki/space1", "feishu"}, + {core.RegionFeishu, "https://feishu.cn/wiki/space1", "larksuite"}, + {core.RegionLark, "https://larksuite.com/wiki/space1", "feishu"}, } for _, c := range cases { @@ -53,22 +54,21 @@ func TestListResources_ResourceURLFollowsRegion(t *testing.T) { // region is the only thing standing between a Lark app and the wrong cloud. func TestClient_ResolvesRegionHostWithoutOverride(t *testing.T) { cases := []struct { - region Region + region core.Region want string }{ - {RegionFeishu, "https://open.feishu.cn"}, - {RegionLark, "https://open.larksuite.com"}, + {core.RegionFeishu, "https://open.feishu.cn"}, + {core.RegionLark, "https://open.larksuite.com"}, } for _, c := range cases { t.Run(c.region.Label, func(t *testing.T) { - cfg, err := parseFeishuConfig(makeConfigNoBaseURL(), c.region) + cfg, err := core.ParseFeishuConfig(makeConfigNoBaseURL(), c.region) if err != nil { - t.Fatalf("parseFeishuConfig: %v", err) + t.Fatalf("ParseFeishuConfig: %v", err) } - client := NewClient(cfg) - if client.baseURL != c.want { - t.Errorf("client baseURL = %q, want %q", client.baseURL, c.want) + if cfg.GetBaseURL() != c.want { + t.Errorf("baseURL = %q, want %q", cfg.GetBaseURL(), c.want) } }) } diff --git a/internal/types/datasource.go b/internal/types/datasource.go index 242318aad..914e6b287 100644 --- a/internal/types/datasource.go +++ b/internal/types/datasource.go @@ -17,7 +17,14 @@ const ( ConnectorTypeFeishu = "feishu" // ConnectorTypeLark is Feishu's international edition (open.larksuite.com). // It shares the Feishu connector; only the API host and tenant differ. - ConnectorTypeLark = "lark" + ConnectorTypeLark = "lark" + // ConnectorTypeFeishuDrive is the Feishu Drive (云盘) mode: syncs documents + // under a user-supplied Drive folder_token, as opposed to a Wiki space. + // Shares the feishu connector package; only resource enumeration + fetch differ. + ConnectorTypeFeishuDrive = "feishu_drive" + // ConnectorTypeLarkDrive is the Lark (international) Drive mode, the + // international counterpart of ConnectorTypeFeishuDrive. + ConnectorTypeLarkDrive = "lark_drive" ConnectorTypeNotion = "notion" ConnectorTypeConfluence = "confluence" ConnectorTypeYuque = "yuque" diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go index 7f5ac047e..2133997ac 100644 --- a/internal/types/knowledge.go +++ b/internal/types/knowledge.go @@ -27,6 +27,8 @@ const ( ChannelWechat = "wechat" // WeChat ChannelWecom = "wecom" // WeCom (企业微信) ChannelFeishu = "feishu" // Feishu / Lark + ChannelFeishuDrive = "feishu_drive" // Feishu Drive (云盘) + ChannelLarkDrive = "lark_drive" // Lark Drive (international) ChannelDingtalk = "dingtalk" // DingTalk ChannelSlack = "slack" // Slack ChannelIM = "im" // Generic IM channel