refactor: use catalog for workflow archive maintenance (#39001)

2026-09-24 23:22:26 +08:00 · 2026-07-20 07:59:29 +00:00
parent f93a5b95c6
commit b7edc5ef0c
10 changed files with 2103 additions and 248 deletions
@@ -513,11 +513,94 @@ class TestArchiveRunIdempotency:
        storage = MagicMock()
        storage.object_exists.return_value = True

-        result = archiver._archive_bundle(MagicMock(), storage, [run])
+        with patch.object(archiver, "_sync_existing_bundle_index") as sync_existing_bundle_index:
+            result = archiver._archive_bundle(MagicMock(), storage, [run])

        assert result.success is True
        assert result.skipped is True
        assert result.error == "bundle already archived"
+        sync_existing_bundle_index.assert_called_once()
+
+    def test_existing_bundle_catalog_publication_failure_is_not_success(self):
+        archiver = WorkflowRunArchiver(days=90)
+        run = _run()
+        session = MagicMock()
+        storage = MagicMock()
+        storage.object_exists.return_value = True
+
+        with patch.object(archiver, "_sync_existing_bundle_index", side_effect=RuntimeError("catalog unavailable")):
+            result = archiver._archive_bundle(session, storage, [run])
+
+        assert result.success is False
+        assert result.error == "catalog unavailable"
+        session.rollback.assert_called_once()
+
+    def test_retry_repairs_index_after_catalog_commit_then_index_write_failure(self):
+        archiver = WorkflowRunArchiver(days=90)
+        run = _run()
+        identity = archiver._build_bundle_identity([run])
+        index_key = archiver._get_index_object_key(identity)
+        manifest_key = archiver._get_manifest_object_key(identity)
+        storage = FakeArchiveStorage()
+        original_put_object = storage.put_object
+        index_write_count = 0
+
+        def put_object(key: str, data: bytes) -> str:
+            nonlocal index_write_count
+            if key == index_key:
+                index_write_count += 1
+                if index_write_count == 2:
+                    raise RuntimeError("index write failed")
+            return original_put_object(key, data)
+
+        storage.put_object = MagicMock(side_effect=put_object)
+        first_session = MagicMock()
+        first_session.scalar.return_value = None
+        table_data = {"workflow_runs": [{"id": run.id, "tenant_id": run.tenant_id}]}
+
+        with (
+            patch.object(archiver, "_lock_runs_for_archive", return_value=[run]),
+            patch.object(archiver, "_extract_bundle_data", return_value=table_data),
+        ):
+            first_result = archiver._archive_bundle(first_session, storage, [run])
+
+        assert first_result.success is False
+        assert first_result.error == "index write failed"
+        assert manifest_key in storage.objects
+        assert json.loads(storage.objects[index_key])["run_ids"] == []
+
+        storage.list_objects = MagicMock(wraps=storage.list_objects)
+        retry_session = MagicMock()
+        retry_session.scalar.return_value = None
+
+        retry_result = archiver._archive_bundle(retry_session, storage, [run])
+
+        assert retry_result.success is True
+        assert retry_result.skipped is True
+        assert json.loads(storage.objects[index_key])["manifest_keys"] == [manifest_key]
+        assert json.loads(storage.objects[index_key])["run_ids"] == [run.id]
+        storage.list_objects.assert_not_called()
+
+    def test_existing_manifest_with_missing_index_fails_without_partial_rebuild(self):
+        archiver = WorkflowRunArchiver(days=90)
+        run = _run()
+        identity = archiver._build_bundle_identity([run])
+        _, _, manifest_data = archiver._build_archive_payload(
+            identity,
+            [run],
+            {"workflow_runs": [{"id": run.id, "tenant_id": run.tenant_id}]},
+        )
+        manifest_key = archiver._get_manifest_object_key(identity)
+        index_key = archiver._get_index_object_key(identity)
+        storage = FakeArchiveStorage({manifest_key: manifest_data})
+        storage.list_objects = MagicMock(wraps=storage.list_objects)
+
+        result = archiver._archive_bundle(MagicMock(), storage, [run])
+
+        assert result.success is False
+        assert "archive shard index missing" in (result.error or "")
+        assert index_key not in storage.objects
+        storage.list_objects.assert_not_called()

    def test_successful_bundle_persists_archive_index(self):
        archiver = WorkflowRunArchiver(days=90)
@@ -550,6 +633,28 @@ class TestArchiveRunIdempotency:
        assert archived_bundle.row_count == 2
        session.commit.assert_called_once()

+    def test_new_bundle_catalog_commit_failure_is_not_success(self):
+        archiver = WorkflowRunArchiver(days=90)
+        run = _run(str(uuid.uuid4()))
+        run.tenant_id = str(uuid.uuid4())
+        session = MagicMock()
+        session.scalar.return_value = None
+        session.commit.side_effect = RuntimeError("catalog commit failed")
+        storage = MagicMock()
+        storage.object_exists.return_value = False
+        storage.list_objects.return_value = []
+        table_data = {"workflow_runs": [{"id": run.id, "tenant_id": run.tenant_id}]}
+
+        with (
+            patch.object(archiver, "_lock_runs_for_archive", return_value=[run]),
+            patch.object(archiver, "_extract_bundle_data", return_value=table_data),
+        ):
+            result = archiver._archive_bundle(session, storage, [run])
+
+        assert result.success is False
+        assert result.error == "catalog commit failed"
+        session.rollback.assert_called_once()
+
    def test_index_skips_all_already_archived_runs(self):
        archiver = WorkflowRunArchiver(days=90)
        run = MagicMock()