opensource/bisheng
3
0
Fork 0
mirror of https://github.com/dataelement/bisheng.git synced 2026-09-01 15:32:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add information limit error code

Browse Source
This commit is contained in:
GuoQing Zhang
2026-04-24 17:41:18 +08:00
parent 0f811d277b
commit daf2de6cb8
3 changed files with 143 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/nginx/conf.d/default.conf
+10 -2
View File
@@ -6,6 +6,14 @@ map $http_upgrade $connection_upgrade {
}
upstream backend_server {
server backend:7860; # backend api
server 192.168.106.115:8098;
}
upstream minio_server {
server minio:9000;
}
server {
gzip on;
@@ -45,7 +53,7 @@ server {
location ~ ^(/workspace)?/api(/|$) {
rewrite ^/workspace(/.*)$ $1 break;
proxy_pass http://backend:7860;
proxy_pass http://backend_server;
proxy_read_timeout 300s;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
@@ -60,6 +68,6 @@ server {
location ~ ^(/workspace)?/bisheng|/tmp-dir {
rewrite ^/workspace(/.*)$ $1 break;
proxy_pass http://minio:9000;
proxy_pass http://minio_server;
}
}
src/backend/bisheng/common/errcode/channel.py
+15
View File
@@ -29,6 +29,21 @@ class InformationSourcePageError(BaseErrorCode):
Msg: str = 'Single article or non-list page detected. Please provide a valid "Column List Page" URL (e.g., News or Policy list pages)'
class InformationSourceCrawlLimitError(BaseErrorCode):
Code: int = 19006
Msg: str = '网站爬取数量超过当前 API key 限制'
class InformationSourceSubscriptionLimitError(BaseErrorCode):
Code: int = 19007
Msg: str = '订阅信息源数量超过当前 API key 限制'
class InformationSourceWechatSearchLimitError(BaseErrorCode):
Code: int = 19008
Msg: str = '公众号检索次数超过当前 API key 限制'
# Channel Management error codes, module code: 190
class ChannelNotFoundError(BaseErrorCode):
Code: int = 19010
src/backend/bisheng/core/external/bisheng_information_client/client.py
Vendored
+118 -116
View File
@@ -5,7 +5,9 @@ import httpx
from aiohttp import ClientTimeout
from bisheng.common.errcode.channel import BishengInformationUnAuthorizedError, BishengInformationServiceError, \
InformationSourceParseError, InformationSourceAuthError, InformationSourcePageError
InformationSourceParseError, InformationSourceAuthError, InformationSourcePageError, \
InformationSourceCrawlLimitError, InformationSourceSubscriptionLimitError, \
InformationSourceWechatSearchLimitError
from bisheng.core.external.bisheng_information_client.response_schema import InformationSourceResponse, \
CrawlWebsiteResponse, InformationArticlesResponse
from bisheng.core.external.http_client.client import AsyncHttpClient
@@ -52,6 +54,81 @@ class BishengInformationClient(object):
return self._api_key()
return self._api_key
@staticmethod
def _raise_limit_error(code: int) -> None:
if code == 10010:
raise InformationSourceCrawlLimitError()
if code == 10011:
raise InformationSourceSubscriptionLimitError()
if code == 10012:
raise InformationSourceWechatSearchLimitError()
@staticmethod
def _raise_parse_error(code: int) -> None:
if code == 10000:
raise InformationSourceParseError()
if code == 10001:
raise InformationSourceAuthError()
if code == 10002:
raise InformationSourcePageError()
def _handle_common_response_code(
self,
response_body: dict,
default_error_message: str,
*,
include_parse_errors: bool = False,
unknown_error_handler: Optional[Callable[[dict], None]] = None,
) -> None:
code = response_body.get("code", -1)
if code == 200:
return
if code == 401:
raise BishengInformationUnAuthorizedError()
self._raise_limit_error(code)
if include_parse_errors:
self._raise_parse_error(code)
if unknown_error_handler:
unknown_error_handler(response_body)
raise BishengInformationServiceError(msg=f"{default_error_message}: {response_body}")
@staticmethod
def _get_response_body(response) -> dict:
if hasattr(response, "body"):
return response.body or {}
return response.json()
def _handle_response(
self,
response,
default_error_message: str,
*,
include_parse_errors: bool = False,
unknown_error_handler: Optional[Callable[[dict], None]] = None,
) -> dict:
if response.status_code != 200:
raise BishengInformationServiceError(
msg=f"{default_error_message}: {response.status_code} - {getattr(response, 'text', None) or self._get_response_body(response)}"
)
response_body = self._get_response_body(response)
self._handle_common_response_code(
response_body,
default_error_message,
include_parse_errors=include_parse_errors,
unknown_error_handler=unknown_error_handler,
)
return response_body
@staticmethod
def _raise_subscribe_error(response, action: str) -> None:
raise InformationSourceSubscribeError(
f"Failed to {action} information source: {response.status_code} - "
f"{getattr(response, 'body', None) or getattr(response, 'text', None)}"
)
async def add_website_information_source(self, url: str) -> InformationSourceResponse:
"""Add a new information source by URL."""
endpoint = f"{self.base_url}/information/add_website"
@@ -59,24 +136,13 @@ class BishengInformationClient(object):
data = {"url": url}
response = await self.http_client.post(endpoint, body=data, headers=headers, timeout=self.timeout)
if response.status_code != 200:
raise BishengInformationServiceError()
response_body = self._handle_response(
response,
"Failed to add website information source",
include_parse_errors=True,
)
code = response.body.get("code", -1)
if code == 401:
raise BishengInformationUnAuthorizedError()
elif code == 10000:
raise InformationSourceParseError()
elif code == 10001:
raise InformationSourceAuthError()
elif code == 10002:
raise InformationSourcePageError()
elif code != 200:
raise BishengInformationServiceError(
msg=f"Failed to list information sources: {response.body}")
information_source_data = InformationSourceResponse.model_validate(response.body.get("data"))
information_source_data = InformationSourceResponse.model_validate(response_body.get("data"))
return information_source_data
@@ -87,24 +153,13 @@ class BishengInformationClient(object):
data = {"url": url}
response = await self.http_client.post(endpoint, body=data, headers=headers, timeout=self.timeout)
if response.status_code != 200:
raise BishengInformationServiceError()
response_body = self._handle_response(
response,
"Failed to add wechat information source",
include_parse_errors=True,
)
code = response.body.get("code", -1)
if code == 401:
raise BishengInformationUnAuthorizedError()
elif code == 10000:
raise InformationSourceParseError()
elif code == 10001:
raise InformationSourceAuthError()
elif code == 10002:
raise InformationSourcePageError()
elif code != 200:
raise BishengInformationServiceError(
msg=f"Failed to list information sources: {response.body}")
information_source_data = InformationSourceResponse.model_validate(response.body.get("data"))
information_source_data = InformationSourceResponse.model_validate(response_body.get("data"))
return information_source_data
@@ -124,20 +179,10 @@ class BishengInformationClient(object):
params["business_type"] = business_type.value
response = await self.http_client.get(endpoint, headers=headers, params=params, timeout=self.timeout)
response_body = self._handle_response(response, "Failed to search information sources")
if response.status_code != 200:
raise BishengInformationServiceError()
code = response.body.get("code", -1)
if code == 401:
raise BishengInformationUnAuthorizedError()
elif code != 200:
raise BishengInformationServiceError(
msg=f"Failed to list information sources: {response.body}")
information_sources_data = response.body.get("data", [])
total = response.body.get("totalCount", 0)
information_sources_data = response_body.get("data", [])
total = response_body.get("totalCount", 0)
return [InformationSourceResponse.model_validate(item) for item in information_sources_data], total
@@ -147,19 +192,9 @@ class BishengInformationClient(object):
headers = {"X-API-Key": self.api_key}
data = {"information_ids": source_ids}
response = await self.http_client.post(endpoint, body=data, headers=headers, timeout=self.timeout)
response_body = self._handle_response(response, "Failed to get information sources by ids")
if response.status_code != 200:
raise BishengInformationServiceError()
code = response.body.get("code", -1)
if code == 401:
raise BishengInformationUnAuthorizedError()
elif code != 200:
raise BishengInformationServiceError(
msg=f"Failed to list information sources: {response.body}")
information_sources_data = response.body.get("data", [])
information_sources_data = response_body.get("data", [])
return [InformationSourceResponse.model_validate(item) for item in information_sources_data]
async def list_information_sources(self, business_type: BusinessType, page: int = 1, page_size: int = 20) -> tuple[
@@ -175,19 +210,10 @@ class BishengInformationClient(object):
}
response = await self.http_client.get(endpoint, headers=headers, params=params, timeout=self.timeout)
response_body = self._handle_response(response, "Failed to list information sources")
if response.status_code != 200:
raise BishengInformationServiceError()
code = response.body.get("code", -1)
if code == 401:
raise BishengInformationUnAuthorizedError()
elif code != 200:
raise BishengInformationServiceError(
msg=f"Failed to list information sources: {response.body}")
information_sources_data = response.body.get("data", [])
total = response.body.get("totalCount", 0)
information_sources_data = response_body.get("data", [])
total = response_body.get("totalCount", 0)
res = [InformationSourceResponse.model_validate(item) for item in information_sources_data]
return res, total
@@ -198,17 +224,11 @@ class BishengInformationClient(object):
headers = {"X-API-Key": self.api_key}
data = {"information_ids": source_ids}
response = await self.http_client.post(endpoint, body=data, headers=headers, timeout=self.timeout)
if response.status_code != 200:
raise InformationSourceSubscribeError(
f"Failed to subscribe to information source: {response.status_code} - {response.error}")
code = response.body.get("code", -1)
if code == 401:
raise BishengInformationUnAuthorizedError()
elif code != 200:
raise InformationSourceSubscribeError(
f"Failed to subscribe to information source: {response.status_code} - {response.error}")
self._handle_response(
response,
"Failed to subscribe to information source",
unknown_error_handler=lambda _: self._raise_subscribe_error(response, "subscribe to"),
)
async def unsubscribe_information_source(self, source_ids: list[str]) -> None:
"""Unsubscribe from an information source by source_id."""
@@ -216,10 +236,11 @@ class BishengInformationClient(object):
headers = {"X-API-Key": self.api_key}
data = {"information_ids": source_ids}
response = await self.http_client.post(endpoint, body=data, headers=headers, timeout=self.timeout)
if response.status_code != 200:
raise InformationSourceSubscribeError(
f"Failed to unsubscribe from information source: {response.status_code} - {response.error}")
self._handle_response(
response,
"Failed to unsubscribe from information source",
unknown_error_handler=lambda _: self._raise_subscribe_error(response, "unsubscribe from"),
)
async def crawl_website(self, url: str) -> CrawlWebsiteResponse:
"""Crawl a website by URL."""
@@ -227,25 +248,13 @@ class BishengInformationClient(object):
headers = {"X-API-Key": self.api_key}
data = {"url": url}
response = await self.http_client.post(endpoint, body=data, headers=headers, timeout=self.timeout)
response_body = self._handle_response(
response,
"Failed to crawl website",
include_parse_errors=True,
)
if response.status_code != 200:
raise BishengInformationServiceError()
code = response.body.get("code", -1)
if code == 401:
raise BishengInformationUnAuthorizedError()
elif code == 10000:
raise InformationSourceParseError()
elif code == 10001:
raise InformationSourceAuthError()
elif code == 10002:
raise InformationSourcePageError()
elif code != 200:
raise BishengInformationServiceError(
msg=f"Failed to list information sources: {response.body}")
result = response.body.get("data", {})
result = response_body.get("data", {})
return CrawlWebsiteResponse.model_validate(result)
@@ -265,14 +274,7 @@ class BishengInformationClient(object):
params["min_create_time"] = min_create_time
with httpx.Client() as client:
response = client.get(endpoint, headers=headers, params=params)
if response.status_code != 200:
raise BishengInformationServiceError(
msg=f"Failed to get information articles: {response.status_code} - {response.text}")
result = response.json()
if result.get("code") != 200:
raise BishengInformationServiceError(
msg=f"Failed to get information articles: {response.status_code} - {response.text}")
return InformationArticlesResponse(information=result.get("data", {}).get("information"),
articles=result.get("data", {}).get("articles", []),
total=result.get("totalCount", 0))
response_body = self._handle_response(response, "Failed to get information articles")
return InformationArticlesResponse(information=response_body.get("data", {}).get("information"),
articles=response_body.get("data", {}).get("articles", []),
total=response_body.get("totalCount", 0))