feat: add portal telemetry counts

This commit is contained in:
Wenruli
2026-08-06 11:03:48 +08:00
parent 3b0969cb09
commit 457badb445
6 changed files with 197 additions and 9 deletions
@@ -0,0 +1,8 @@
KNOWLEDGE_SPACE_CONTENT_STAT_INDEX = "mid_knowledge_space_content_stat"
KNOWLEDGE_SPACE_DASHBOARD_FILE_LEVELS = (
"public",
"department",
"team",
"team_ks",
)
@@ -2,6 +2,10 @@ import logging
from typing import Any
from bisheng.common.constants.enums.telemetry import BaseTelemetryTypeEnum
from bisheng.common.constants.telemetry import (
KNOWLEDGE_SPACE_CONTENT_STAT_INDEX,
KNOWLEDGE_SPACE_DASHBOARD_FILE_LEVELS,
)
from bisheng.common.schemas.telemetry.event_data_schema import (
PortalDocumentDownloadEventData,
PortalDocumentReadEventData,
@@ -223,6 +227,40 @@ class PortalTelemetryEventService:
"qa_count": counts[BaseTelemetryTypeEnum.PORTAL_QA.value],
}
@staticmethod
async def count_dashboard_files() -> int:
"""Count file snapshots with the global dashboard metric definition."""
es_client = await get_statistics_es_connection()
response = await es_client.search(
index=KNOWLEDGE_SPACE_CONTENT_STAT_INDEX,
body={
"size": 0,
"query": {
"bool": {
"filter": [
{"term": {"record_type": "file"}},
{"term": {"file_type": 1}},
{
"terms": {
"space_level": list(KNOWLEDGE_SPACE_DASHBOARD_FILE_LEVELS),
}
},
]
}
},
"aggs": {
"total_file_count": {
"value_count": {
"field": "file_id",
}
}
},
},
filter_path="aggregations.total_file_count.value",
)
value = response["aggregations"]["total_file_count"]["value"]
return int(value or 0)
def is_portal_bff_proxy_source(source: str | None) -> bool:
return (source or "").strip() == PORTAL_BFF_TELEMETRY_SOURCE
@@ -22,7 +22,6 @@ from bisheng.knowledge.api.dependencies import (
get_portal_pdf_download_service,
)
from bisheng.knowledge.api.portal_pdf_download_response import prepare_portal_pdf_download_response
from bisheng.knowledge.domain.models.knowledge_file import KnowledgeFileDao
from bisheng.knowledge.domain.schemas.knowledge_space_schema import (
ChatReq,
KnowledgeSpaceFolderStatsReq,
@@ -357,7 +356,7 @@ async def get_shougang_portal_home_stats(
_ = login_user
result, total_files = await asyncio.gather(
PortalTelemetryEventService.count_home_events(),
KnowledgeFileDao.async_count_all_success_files(),
PortalTelemetryEventService.count_dashboard_files(),
)
return resp_200(ShougangPortalHomeStatsResp(**result, total_files=total_files).model_dump(mode="json"))
@@ -11,6 +11,7 @@ from elasticsearch import helpers
from loguru import logger
from pydantic import BaseModel, Field
from bisheng.common.constants.telemetry import KNOWLEDGE_SPACE_CONTENT_STAT_INDEX
from bisheng.common.schemas.telemetry.base_telemetry_schema import UserDepartmentInfo
from bisheng.core.cache.redis_manager import get_redis_client, get_redis_client_sync
from bisheng.knowledge.domain.constants import (
@@ -81,8 +82,8 @@ class ProjectionWorkItem:
class KnowledgeSpaceContentStat(BaseMidTable):
INDEX_NAME: ClassVar[str] = "mid_knowledge_space_content_stat"
_index_name: str = "mid_knowledge_space_content_stat"
INDEX_NAME: ClassVar[str] = KNOWLEDGE_SPACE_CONTENT_STAT_INDEX
_index_name: str = KNOWLEDGE_SPACE_CONTENT_STAT_INDEX
_update_mappings_on_existing: bool = True
_include_common_mappings: bool = False
_refresh_settings_applied: ClassVar[set[str]] = set()
@@ -1,3 +1,7 @@
from bisheng.common.constants.telemetry import (
KNOWLEDGE_SPACE_CONTENT_STAT_INDEX,
KNOWLEDGE_SPACE_DASHBOARD_FILE_LEVELS,
)
from bisheng.core.database import get_async_db_session, get_database_connection
from bisheng.database.models.group import DefaultGroup
from bisheng.database.models.group_resource import GroupResourceDao, GroupResource, ResourceTypeEnum
@@ -719,8 +723,8 @@ DASHBOARD_DATASET = [
),
DashboardDataset(
dataset_name="知识空间内容统计",
dataset_code="mid_knowledge_space_content_stat",
es_index_name="mid_knowledge_space_content_stat",
dataset_code=KNOWLEDGE_SPACE_CONTENT_STAT_INDEX,
es_index_name=KNOWLEDGE_SPACE_CONTENT_STAT_INDEX,
description="知识空间成功文件与文件预览统计数据表",
is_commercial_only=False,
schema_config=SchemaConfig(
@@ -734,7 +738,7 @@ DASHBOARD_DATASET = [
TermOp(field="file_type", value=1),
TermsOp(
field="space_level",
value=["public", "department", "team", "team_ks"],
value=list(KNOWLEDGE_SPACE_DASHBOARD_FILE_LEVELS),
),
]),
aggregations=[
@@ -762,7 +766,7 @@ DASHBOARD_DATASET = [
TermOp(field="file_type", value=1),
TermsOp(
field="space_level",
value=["public", "department", "team", "team_ks"],
value=list(KNOWLEDGE_SPACE_DASHBOARD_FILE_LEVELS),
),
]),
aggregations=[
@@ -781,7 +785,7 @@ DASHBOARD_DATASET = [
TermOp(field="record_type", value="file"),
TermsOp(
field="space_level",
value=["public", "department", "team", "team_ks"],
value=list(KNOWLEDGE_SPACE_DASHBOARD_FILE_LEVELS),
),
]),
aggregations=[
@@ -0,0 +1,138 @@
from types import SimpleNamespace
import pytest
from bisheng.common.telemetry import portal_event_service
from bisheng.knowledge.api.endpoints import shougang_portal
class _FakeSearchClient:
def __init__(self, *, total: int = 0) -> None:
self.total = total
self.search_calls: list[dict] = []
async def search(self, **kwargs):
self.search_calls.append(kwargs)
return {
"aggregations": {
"total_file_count": {
"value": self.total,
}
}
}
@pytest.mark.asyncio
async def test_portal_file_count_uses_dashboard_es_metric(monkeypatch: pytest.MonkeyPatch):
fake_client = _FakeSearchClient(total=41)
async def fake_get_es_connection():
return fake_client
monkeypatch.setattr(
portal_event_service,
"get_statistics_es_connection",
fake_get_es_connection,
)
result = await portal_event_service.PortalTelemetryEventService.count_dashboard_files()
assert result == 41
assert fake_client.search_calls == [
{
"index": "mid_knowledge_space_content_stat",
"body": {
"size": 0,
"query": {
"bool": {
"filter": [
{"term": {"record_type": "file"}},
{"term": {"file_type": 1}},
{
"terms": {
"space_level": [
"public",
"department",
"team",
"team_ks",
]
}
},
]
}
},
"aggs": {
"total_file_count": {
"value_count": {
"field": "file_id",
}
}
},
},
"filter_path": "aggregations.total_file_count.value",
}
]
@pytest.mark.asyncio
async def test_portal_home_stats_uses_es_file_count(monkeypatch: pytest.MonkeyPatch):
async def fake_count_home_events():
return {
"read_count": 12,
"favorite_count": 3,
"qa_count": 7,
}
async def fake_count_dashboard_files():
return 41
monkeypatch.setattr(
shougang_portal.PortalTelemetryEventService,
"count_home_events",
fake_count_home_events,
)
monkeypatch.setattr(
shougang_portal.PortalTelemetryEventService,
"count_dashboard_files",
fake_count_dashboard_files,
)
response = await shougang_portal.get_shougang_portal_home_stats(
login_user=SimpleNamespace(user_id=1),
)
assert response.data == {
"read_count": 12,
"favorite_count": 3,
"qa_count": 7,
"total_files": 41,
}
@pytest.mark.asyncio
async def test_portal_home_stats_propagates_es_file_count_failure(monkeypatch: pytest.MonkeyPatch):
async def fake_count_home_events():
return {
"read_count": 12,
"favorite_count": 3,
"qa_count": 7,
}
async def fail_count_dashboard_files():
raise RuntimeError("statistics es unavailable")
monkeypatch.setattr(
shougang_portal.PortalTelemetryEventService,
"count_home_events",
fake_count_home_events,
)
monkeypatch.setattr(
shougang_portal.PortalTelemetryEventService,
"count_dashboard_files",
fail_count_dashboard_files,
)
with pytest.raises(RuntimeError, match="statistics es unavailable"):
await shougang_portal.get_shougang_portal_home_stats(
login_user=SimpleNamespace(user_id=1),
)