opensource/WeKnora
3
0
Fork 0
mirror of https://github.com/Tencent/WeKnora.git synced 2026-10-28 03:29:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(datasource): keep the source document's last-modified time through ingestion

Browse Source 
Connectors already fill FetchedItem.UpdatedAt (Feishu from the real edit
time), but ingestItem dropped it: the persisted metadata only carried
external_id, source_resource_id and datasource_id, and the knowledge
row's own UpdatedAt moves on every re-parse, so nothing downstream could
tell how old a document is.

Write it into metadata as source_updated_at (RFC3339, UTC) when the
connector supplied one, and leave the key out otherwise so an absent
time stays distinguishable from a zero one.

Refs #3016
This commit is contained in:
po-et
2026-09-07 11:52:56 +08:00
committed by lyingbug
parent 801005f736
commit 160c806b94
2 changed files with 38 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/application/service/datasource_service.go
+6
View File
@@ -1251,6 +1251,12 @@ func (s *DataSourceService) ingestItem(ctx context.Context, ds *types.DataSource
"source_resource_id": item.SourceResourceID,
"datasource_id": ds.ID,
}
// The source system's own last-modified time, when the connector supplied
// one. The knowledge row's UpdatedAt moves on every re-parse, so this is
// the only record of how old the document itself is.
if !item.UpdatedAt.IsZero() {
metadata["source_updated_at"] = item.UpdatedAt.UTC().Format(time.RFC3339)
}
for k, v := range item.Metadata {
metadata[k] = v
}
internal/application/service/datasource_service_test.go
+32
View File
@@ -501,6 +501,38 @@ func TestIngestItem_URLCreationAttachesDataSourceMetadata(t *testing.T) {
assert.Equal(t, "url:1", repo.metadataUpdates[0]["external_id"])
}
// TestIngestItem_PersistsSourceUpdatedAt verifies that the connector-supplied
// last-modified time reaches the persisted metadata as an RFC3339 UTC string,
// and that an item without one gets no key rather than a zero time.
func TestIngestItem_PersistsSourceUpdatedAt(t *testing.T) {
ds := &types.DataSource{ID: "ds-1", TenantID: 1, KnowledgeBaseID: "kb-1"}
edited := time.Date(2023, 4, 5, 6, 7, 8, 0, time.FixedZone("CST", 8*3600))
repo := &deletionLookupKnowledgeRepo{}
ks := &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-1"}}
svc := &DataSourceService{knowledgeService: ks}
_, err := svc.ingestItem(context.Background(), ds, &types.FetchedItem{
ExternalID: "url:1",
URL: "https://example.com/doc",
UpdatedAt: edited,
}, nil)
require.NoError(t, err)
require.Len(t, repo.metadataUpdates, 1)
assert.Equal(t, "2023-04-04T22:07:08Z", repo.metadataUpdates[0]["source_updated_at"])
repo = &deletionLookupKnowledgeRepo{}
ks = &sweepFakeKS{repo: repo, createURLKnowledge: &types.Knowledge{ID: "url-knowledge-2"}}
svc = &DataSourceService{knowledgeService: ks}
_, err = svc.ingestItem(context.Background(), ds, &types.FetchedItem{
ExternalID: "url:2",
URL: "https://example.com/doc2",
}, nil)
require.NoError(t, err)
require.Len(t, repo.metadataUpdates, 1)
_, present := repo.metadataUpdates[0]["source_updated_at"]
assert.False(t, present)
}
func TestProcessSync_SyncDeletionsDeletesMatchingKnowledge(t *testing.T) {
h := newSyncDeletionHarness(t, true, "ds-delete-characterization", "log-delete-characterization", nil, nil)
updated, err := h.run(t)