From 3e8c0fdcef321edfba7aeed2cf22b5e1bfcd0e52 Mon Sep 17 00:00:00 2001 From: Likalikali Date: Fri, 7 Aug 2026 20:40:12 +0800 Subject: [PATCH] chore: dep inject for model in website crawl api (#40156) --- api/controllers/console/datasets/website.py | 17 ++--- .../console/datasets/test_website.py | 71 +++++-------------- 2 files changed, 25 insertions(+), 63 deletions(-) diff --git a/api/controllers/console/datasets/website.py b/api/controllers/console/datasets/website.py index 39be3f3ce5f..9411d128195 100644 --- a/api/controllers/console/datasets/website.py +++ b/api/controllers/console/datasets/website.py @@ -1,13 +1,12 @@ from typing import Any, Literal -from flask import request from flask_restx import Resource from pydantic import BaseModel, RootModel from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models from controllers.console import console_ns from controllers.console.datasets.error import WebsiteCrawlError -from controllers.console.wraps import account_initialization_required, setup_required +from controllers.console.wraps import account_initialization_required, model_validate, setup_required from libs.login import login_required from services.website_service import WebsiteCrawlApiRequest, WebsiteCrawlStatusApiRequest, WebsiteService @@ -40,12 +39,11 @@ class WebsiteCrawlApi(Resource): @setup_required @login_required @account_initialization_required - def post(self): - payload = WebsiteCrawlPayload.model_validate(console_ns.payload or {}) - + @model_validate(WebsiteCrawlPayload) + def post(self, req_data: WebsiteCrawlPayload): # Create typed request and validate try: - api_request = WebsiteCrawlApiRequest.from_args(payload.model_dump()) + api_request = WebsiteCrawlApiRequest.from_args(req_data.model_dump()) except ValueError as e: raise WebsiteCrawlError(str(e)) @@ -69,12 +67,11 @@ class WebsiteCrawlStatusApi(Resource): @setup_required @login_required @account_initialization_required - def get(self, job_id: str): - args = WebsiteCrawlStatusQuery.model_validate(request.args.to_dict()) - + @model_validate(WebsiteCrawlStatusQuery) + def get(self, req_data: WebsiteCrawlStatusQuery, job_id: str): # Create typed request and validate try: - api_request = WebsiteCrawlStatusApiRequest.from_args(args.model_dump(), job_id) + api_request = WebsiteCrawlStatusApiRequest.from_args(req_data.model_dump(), job_id) except ValueError as e: raise WebsiteCrawlError(str(e)) diff --git a/api/tests/unit_tests/controllers/console/datasets/test_website.py b/api/tests/unit_tests/controllers/console/datasets/test_website.py index 5c7b857c20e..b790059f0c7 100644 --- a/api/tests/unit_tests/controllers/console/datasets/test_website.py +++ b/api/tests/unit_tests/controllers/console/datasets/test_website.py @@ -1,14 +1,15 @@ -from unittest.mock import Mock, PropertyMock, patch +from unittest.mock import Mock import pytest from flask import Flask from pytest_mock import MockerFixture -from controllers.console import console_ns from controllers.console.datasets.error import WebsiteCrawlError from controllers.console.datasets.website import ( WebsiteCrawlApi, + WebsiteCrawlPayload, WebsiteCrawlStatusApi, + WebsiteCrawlStatusQuery, ) from services.website_service import ( WebsiteCrawlApiRequest, @@ -58,16 +59,9 @@ class TestWebsiteCrawlApi: "url": "https://example.com", "options": {"depth": 1}, } + req_data = WebsiteCrawlPayload.model_validate(payload) - with ( - app.test_request_context("/", json=payload), - patch.object( - type(console_ns), - "payload", - new_callable=PropertyMock, - return_value=payload, - ), - ): + with app.test_request_context("/", json=payload): mock_request = Mock(spec=WebsiteCrawlApiRequest) mocker.patch.object( WebsiteCrawlApiRequest, @@ -81,7 +75,7 @@ class TestWebsiteCrawlApi: return_value={"job_id": "job-1"}, ) - result, status = method(api) + result, status = method(api, req_data) assert status == 200 assert result["job_id"] == "job-1" @@ -95,16 +89,9 @@ class TestWebsiteCrawlApi: "url": "bad-url", "options": {}, } + req_data = WebsiteCrawlPayload.model_validate(payload) - with ( - app.test_request_context("/", json=payload), - patch.object( - type(console_ns), - "payload", - new_callable=PropertyMock, - return_value=payload, - ), - ): + with app.test_request_context("/", json=payload): mocker.patch.object( WebsiteCrawlApiRequest, "from_args", @@ -112,7 +99,7 @@ class TestWebsiteCrawlApi: ) with pytest.raises(WebsiteCrawlError, match="invalid payload"): - method(api) + method(api, req_data) def test_crawl_service_error(self, app: Flask, mocker: MockerFixture): api = WebsiteCrawlApi() @@ -123,16 +110,9 @@ class TestWebsiteCrawlApi: "url": "https://example.com", "options": {}, } + req_data = WebsiteCrawlPayload.model_validate(payload) - with ( - app.test_request_context("/", json=payload), - patch.object( - type(console_ns), - "payload", - new_callable=PropertyMock, - return_value=payload, - ), - ): + with app.test_request_context("/", json=payload): mock_request = Mock(spec=WebsiteCrawlApiRequest) mocker.patch.object( WebsiteCrawlApiRequest, @@ -147,7 +127,7 @@ class TestWebsiteCrawlApi: ) with pytest.raises(WebsiteCrawlError, match="crawl failed"): - method(api) + method(api, req_data) class TestWebsiteCrawlStatusApi: @@ -156,14 +136,9 @@ class TestWebsiteCrawlStatusApi: method = unwrap(api.get) job_id = "job-123" - args = {"provider": "firecrawl"} + req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"}) with app.test_request_context("/?provider=firecrawl"): - mocker.patch( - "controllers.console.datasets.website.request.args.to_dict", - return_value=args, - ) - mock_request = Mock(spec=WebsiteCrawlStatusApiRequest) mocker.patch.object( WebsiteCrawlStatusApiRequest, @@ -177,7 +152,7 @@ class TestWebsiteCrawlStatusApi: return_value={"status": "completed"}, ) - result, status = method(api, job_id) + result, status = method(api, req_data, job_id) assert status == 200 assert result["status"] == "completed" @@ -187,14 +162,9 @@ class TestWebsiteCrawlStatusApi: method = unwrap(api.get) job_id = "job-123" - args = {"provider": "firecrawl"} + req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"}) with app.test_request_context("/?provider=firecrawl"): - mocker.patch( - "controllers.console.datasets.website.request.args.to_dict", - return_value=args, - ) - mocker.patch.object( WebsiteCrawlStatusApiRequest, "from_args", @@ -202,21 +172,16 @@ class TestWebsiteCrawlStatusApi: ) with pytest.raises(WebsiteCrawlError, match="invalid provider"): - method(api, job_id) + method(api, req_data, job_id) def test_get_status_service_error(self, app: Flask, mocker: MockerFixture): api = WebsiteCrawlStatusApi() method = unwrap(api.get) job_id = "job-123" - args = {"provider": "firecrawl"} + req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"}) with app.test_request_context("/?provider=firecrawl"): - mocker.patch( - "controllers.console.datasets.website.request.args.to_dict", - return_value=args, - ) - mock_request = Mock(spec=WebsiteCrawlStatusApiRequest) mocker.patch.object( WebsiteCrawlStatusApiRequest, @@ -231,4 +196,4 @@ class TestWebsiteCrawlStatusApi: ) with pytest.raises(WebsiteCrawlError, match="status lookup failed"): - method(api, job_id) + method(api, req_data, job_id)