From 0c44ee94143e0fc121c641b3679672463a6b91ae Mon Sep 17 00:00:00 2001 From: Pao-Sheng Wang Date: Thu, 20 Feb 2025 18:15:31 +0800 Subject: [PATCH] tmp --- wren-ai-service/eval/evaluation.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/wren-ai-service/eval/evaluation.py b/wren-ai-service/eval/evaluation.py index 5254a9038..c1bf14668 100644 --- a/wren-ai-service/eval/evaluation.py +++ b/wren-ai-service/eval/evaluation.py @@ -82,8 +82,10 @@ class Evaluator: try: test_case = LLMTestCase(**formatter(prediction, meta)) - result = evaluate([test_case], self._metrics, ignore_errors=True).test_results[0] - self._score_metrics(test_case, result) + result = evaluate( + [test_case], self._metrics, ignore_errors=True + ).test_results[0] + self._score_metrics(test_case, result, prediction) [metric.collect(test_case, result) for metric in self._post_metrics] except Exception: self._failed_count += 1 @@ -91,11 +93,15 @@ class Evaluator: self._average_score(meta) - def _score_metrics(self, test_case: LLMTestCase, result: TestResult) -> None: + def _score_metrics( + self, test_case: LLMTestCase, result: TestResult, prediction: dict + ) -> None: for metric in result.metrics_data: name = metric.name score = metric.score or 0 + prediction["scores"][name] = score + self._langfuse.score( trace_id=test_case.additional_metadata["trace_id"], name=name, @@ -109,6 +115,8 @@ class Evaluator: self._score_collector[name].append(score) + prediction["status"] = "success" + @observe(name="Summary Trace", capture_input=False, capture_output=False) def _average_score(self, meta: dict) -> None: langfuse_context.update_current_trace(