diff --git a/wren-ai-service/eval/.env.example b/wren-ai-service/eval/.env.example index 7203c02f0..ba66f6ab5 100644 --- a/wren-ai-service/eval/.env.example +++ b/wren-ai-service/eval/.env.example @@ -1,6 +1,8 @@ +LANGFUSE_PROJECT_ID= OPENAI_API_KEY= WREN_IBIS_ENDPOINT=http://localhost:8000 WREN_ENGINE_ENDPOINT=http://localhost:8080 +WREN_IBIS_TIMEOUT=10 bigquery.project-id= bigquery.dataset-id= bigquery.credentials-key= diff --git a/wren-ai-service/eval/data_curation/utils.py b/wren-ai-service/eval/data_curation/utils.py index a78f25c65..490a659db 100644 --- a/wren-ai-service/eval/data_curation/utils.py +++ b/wren-ai-service/eval/data_curation/utils.py @@ -3,6 +3,7 @@ import base64 import os import sys import uuid +from copy import deepcopy from datetime import datetime from pathlib import Path from typing import List, Tuple @@ -126,6 +127,8 @@ async def get_contexts_from_sqls( def get_documents_given_contexts( contexts_list: list[list[str]], mdl_json: dict ) -> list[list[dict]]: + mdl_json_cloned = deepcopy(mdl_json) + def _build_partial_mdl_json( contexts_list: list[list[str]], mdl_json: dict ) -> list[dict]: @@ -218,7 +221,7 @@ def get_documents_given_contexts( return new_mdl_jsons - new_mdl_jsons = _build_partial_mdl_json(contexts_list, mdl_json) + new_mdl_jsons = _build_partial_mdl_json(contexts_list, mdl_json_cloned) return [ { @@ -363,4 +366,4 @@ def get_eval_dataset_in_toml_string(mdl: dict, dataset: list) -> str: doc.add("mdl", mdl) doc.add("eval_dataset", dataset) - return tomlkit.dumps(doc) + return tomlkit.dumps(doc, sort_keys=True) diff --git a/wren-ai-service/eval/evaluation.py b/wren-ai-service/eval/evaluation.py index 7d0bd181a..3be8771f2 100644 --- a/wren-ai-service/eval/evaluation.py +++ b/wren-ai-service/eval/evaluation.py @@ -72,8 +72,8 @@ class Evaluator: self._average_score(meta) def _score_metrics(self, test_case: LLMTestCase, result: TestResult) -> None: - for metric in result.metrics_metadata: - name = metric.metric + for metric in result.metrics_data: + name = metric.name score = metric.score or 0 self._langfuse.score( @@ -137,3 +137,8 @@ if __name__ == "__main__": evaluator.eval(meta, predictions) langfuse_context.flush() + + if meta["langfuse_url"]: + print( + f"\n\nYou can view the evaluation result in Langfuse at {meta['langfuse_url']}/sessions/{meta['session_id']}" + ) diff --git a/wren-ai-service/eval/metrics/column.py b/wren-ai-service/eval/metrics/column.py index 23f221204..39d940c79 100644 --- a/wren-ai-service/eval/metrics/column.py +++ b/wren-ai-service/eval/metrics/column.py @@ -11,7 +11,7 @@ from eval.utils import get_contexts_from_sql, get_data_from_wren_engine class AccuracyMetric(BaseMetric): def __init__(self, engine_config: dict): - self.threshold = 10 + self.threshold = 0 self.score = 0 self._engine_config = engine_config @@ -19,25 +19,50 @@ class AccuracyMetric(BaseMetric): return asyncio.run(self.a_measure(test_case)) def is_subset(self, expected: pd.DataFrame, actual: pd.DataFrame) -> bool: - if not set(actual.columns).issubset(set(expected.columns)): + if not set(expected.columns).issubset(set(actual.columns)): return False - common_columns = actual.columns - if common_columns.empty: - return False + common_columns = sorted(expected.columns) - expected_sorted = expected[sorted(common_columns)] - actual_sorted = actual[sorted(common_columns)] + expected_sorted = expected[common_columns] + actual_sorted = actual[common_columns] + # Ensure that the data types are the same + actual_sorted = actual_sorted.astype(expected_sorted.dtypes.to_dict()) merged = pd.merge( actual_sorted, expected_sorted, - on=list(common_columns), + on=common_columns, how="left", indicator=True, ) return all(merged["_merge"] == "both") + def count_partial_matches( + self, expected: pd.DataFrame, actual: pd.DataFrame + ) -> int: + intersection = set(expected.columns).intersection(set(actual.columns)) + common_columns = sorted(intersection) + if not common_columns: + return 0 + + expected_sorted = expected[common_columns] + actual_sorted = actual[common_columns] + # Ensure that the data types are the same + actual_sorted = actual_sorted.astype(expected_sorted.dtypes.to_dict()) + + merged = pd.merge( + actual_sorted, + expected_sorted, + on=common_columns, + how="left", + indicator=True, + ) + if all(merged["_merge"] == "both"): + return len(intersection) / len(expected.columns) + else: + return 0 + async def _retrieve_data(self, sql: str) -> pd.DataFrame: response = await get_data_from_wren_engine(sql=sql, **self._engine_config) df = pd.DataFrame(**response) @@ -49,12 +74,17 @@ class AccuracyMetric(BaseMetric): expected_dataset = await self._retrieve_data(test_case.expected_output) actual_dataset = await self._retrieve_data(test_case.actual_output) + print(f"expected columns: {set(expected_dataset.columns)}") + print(f"actual columns: {set(actual_dataset.columns)}") + if expected_dataset.equals(actual_dataset) or self.is_subset( expected_dataset, actual_dataset ): self.success = True self.score = 1 return self.score + + self.score = self.count_partial_matches(expected_dataset, actual_dataset) except Exception as e: self.error = f"Error occurred while evaluating the metric: {e}" traceback.print_exc() @@ -78,13 +108,13 @@ class AccuracyMultiCandidateMetric(BaseMetric): self._questions = {} def collect(self, test_case: LLMTestCase, result: TestResult): - for metric in result.metrics_metadata: - if metric.metric != "Accuracy(column-based)": + for metric in result.metrics_data: + if metric.name != "Accuracy(column-based)": continue # or 0 to avoid when metric.error is exist self._questions[test_case.input] = ( - self._questions.get(test_case.input, False) or metric.score or 0 > 0 + self._questions.get(test_case.input, 0) or metric.score or 0 ) def measure(self): diff --git a/wren-ai-service/eval/pipelines.py b/wren-ai-service/eval/pipelines.py index e3d786c56..a54678e80 100644 --- a/wren-ai-service/eval/pipelines.py +++ b/wren-ai-service/eval/pipelines.py @@ -33,7 +33,7 @@ from src.pipelines.ask import generation, retrieval from src.pipelines.indexing import indexing -def deploy_model(mdl: str, pipe) -> None: +def deploy_model(mdl: str, pipe: indexing.Indexing) -> None: async def wrapper(): await pipe.run(orjson.dumps(mdl).decode()) @@ -98,7 +98,7 @@ class Eval: return self._candidate_size def predict(self, queries: list) -> List[Dict[str, Any]]: - def split(batch_size: int) -> list[list]: + def split(queries: list, batch_size: int) -> list[list]: return [ queries[i : i + batch_size] for i in range(0, len(queries), batch_size) ] @@ -109,7 +109,9 @@ class Eval: await asyncio.sleep(self._batch_interval) return [prediction for predictions in results for prediction in predictions] - batches = [asyncio.run(wrapper(batch)) for batch in split(self._batch_size)] + batches = [ + asyncio.run(wrapper(batch)) for batch in split(queries, self._batch_size) + ] return [prediction for batch in batches for prediction in batch] @abstractmethod @@ -174,6 +176,8 @@ class RetrievalPipeline(Eval): document_store_provider: DocumentStoreProvider, **kwargs, ): + super().__init__(meta) + document_store_provider.get_store(recreate_index=True) _indexing = indexing.Indexing( embedder_provider=embedder_provider, @@ -181,7 +185,6 @@ class RetrievalPipeline(Eval): ) deploy_model(mdl, _indexing) - super().__init__(meta) self._retrieval = retrieval.Retrieval( embedder_provider=embedder_provider, document_store_provider=document_store_provider, @@ -288,13 +291,14 @@ class AskPipeline(Eval): engine: Engine, **kwargs, ): + super().__init__(meta, 3) + document_store_provider.get_store(recreate_index=True) _indexing = indexing.Indexing( embedder_provider=embedder_provider, document_store_provider=document_store_provider, ) deploy_model(mdl, _indexing) - super().__init__(meta, 3) self._mdl = mdl self._retrieval = retrieval.Retrieval( @@ -386,7 +390,9 @@ def metrics_initiator(pipeline: str, mdl: dict) -> dict: "dataset_id": os.getenv("bigquery.dataset-id"), "credentials": os.getenv("bigquery.credentials-key"), }, - "timeout": 10, + "timeout": int(os.getenv("WREN_IBIS_TIMEOUT")) + if os.getenv("WREN_IBIS_TIMEOUT") + else 10, "limit": 10, } diff --git a/wren-ai-service/eval/prediction.py b/wren-ai-service/eval/prediction.py index b13dd80c1..79694af43 100644 --- a/wren-ai-service/eval/prediction.py +++ b/wren-ai-service/eval/prediction.py @@ -31,7 +31,13 @@ def generate_meta( embedder_provider: EmbedderProvider, **kwargs, ) -> Dict[str, Any]: + if langfuse_project_id := os.getenv("LANGFUSE_PROJECT_ID"): + langfuse_url = f'{os.getenv("LANGFUSE_HOST", "https://cloud.langfuse.com").rstrip('/')}/project/{langfuse_project_id}' + else: + langfuse_url = "" + return { + "langfuse_url": langfuse_url, "user_id": "wren-evaluator", # this property is using for langfuse "session_id": f"eval_{pipe}_{uuid.uuid4()}", "date": datetime.now(), @@ -47,11 +53,14 @@ def generate_meta( } -def write_prediction(meta, predictions, dir_path="outputs/predictions") -> None: +def write_prediction( + meta: dict, predictions: list[dict], dir_path: str = "outputs/predictions" +) -> None: if Path(dir_path).exists() is False: Path(dir_path).mkdir(parents=True, exist_ok=True) - output_path = f"{dir_path}/prediction_{meta['session_id']}_{meta['date'].strftime("%Y_%m_%d_%H%M%S")}.toml" + output_file = f"prediction_{meta['session_id']}_{meta['date'].strftime("%Y_%m_%d_%H%M%S")}.toml" + output_path = f"{dir_path}/{output_file}" doc = document() doc.add("meta", meta) @@ -60,7 +69,10 @@ def write_prediction(meta, predictions, dir_path="outputs/predictions") -> None: with open(output_path, "w") as file: file.write(dumps(doc)) - print(f"Prediction result is saved at {output_path}") + print(f"\n\nPrediction result is saved at {output_path}") + print( + f"You can then evaluate the prediction result by running `just eval {output_file}`" + ) def obtain_commit_hash() -> str: @@ -119,7 +131,7 @@ def parse_args() -> Tuple[str]: if __name__ == "__main__": - path, pipe = parse_args() + path, pipe_name = parse_args() dotenv.load_dotenv() utils.load_env_vars() @@ -128,9 +140,9 @@ if __name__ == "__main__": dataset = parse_toml(path) providers = init_providers(dataset["mdl"]) - meta = generate_meta(path=path, dataset=dataset, pipe=pipe, **providers) + meta = generate_meta(path=path, dataset=dataset, pipe=pipe_name, **providers) - pipe = pipelines.init(pipe, meta, mdl=dataset["mdl"], providers=providers) + pipe = pipelines.init(pipe_name, meta, mdl=dataset["mdl"], providers=providers) predictions = pipe.predict(dataset["eval_dataset"]) meta["expected_batch_size"] = meta["query_count"] * pipe.candidate_size @@ -138,3 +150,8 @@ if __name__ == "__main__": write_prediction(meta, predictions) langfuse_context.flush() + + if meta["langfuse_url"]: + print( + f"You can also view the prediction result in Langfuse at {meta['langfuse_url']}/sessions/{meta['session_id']}" + ) diff --git a/wren-ai-service/eval/utils.py b/wren-ai-service/eval/utils.py index ef273c072..1b8d10e05 100644 --- a/wren-ai-service/eval/utils.py +++ b/wren-ai-service/eval/utils.py @@ -46,7 +46,7 @@ async def get_data_from_wren_engine( return {"data": [], "columns": []} data = await response.json() - column_names = [f"{i}_{col}" for i, col in enumerate(data["columns"])] + column_names = [col for col in data["columns"]] return {"data": data["data"], "columns": column_names} @@ -162,7 +162,9 @@ def trace_metadata( type: TRACE_TYPES, ) -> dict: if type not in get_args(TRACE_TYPES): - raise ValueError("Invalid type") + raise ValueError( + f"Invalid type: {type}, should be one of {get_args(TRACE_TYPES)}" + ) return { "commit": meta["commit"], "dataset_id": meta["dataset_id"], diff --git a/wren-ai-service/poetry.lock b/wren-ai-service/poetry.lock index d663ebdae..c1976cfad 100644 --- a/wren-ai-service/poetry.lock +++ b/wren-ai-service/poetry.lock @@ -785,24 +785,25 @@ vision = ["Pillow (>=9.4.0)"] [[package]] name = "deepeval" -version = "0.21.68" +version = "1.0.6" description = "The open-source evaluation framework for LLMs." optional = false python-versions = "*" files = [ - {file = "deepeval-0.21.68-py3-none-any.whl", hash = "sha256:eeff7d707f849eef223c560246f04483d4a9bf470170bda772c858d4c88983ea"}, - {file = "deepeval-0.21.68.tar.gz", hash = "sha256:e4c6011952ed6cad014fc341f50a8813966001d7a917616b875089121dc2eafb"}, + {file = "deepeval-1.0.6-py3-none-any.whl", hash = "sha256:f57da0db421c295c3d8222b2ccd1465236efaecc11dc1b8525ef674baf30cd41"}, + {file = "deepeval-1.0.6.tar.gz", hash = "sha256:15ec4d4ec0f6633216f57bbb0e4cc9d9270238b551c66d19a7007fabc4c46467"}, ] [package.dependencies] docx2txt = ">=0.8,<1.0" +grpcio = "1.63.0" importlib-metadata = ">=6.0.2" langchain = "*" langchain-core = "*" langchain-openai = "*" -opentelemetry-api = ">=1.14.0,<2.0.0" -opentelemetry-exporter-otlp-proto-grpc = ">=1.24.0,<2.0.0" -opentelemetry-sdk = ">=1.14.0,<2.0.0" +opentelemetry-api = "1.25.0" +opentelemetry-exporter-otlp-proto-grpc = "1.25.0" +opentelemetry-sdk = "1.25.0" portalocker = "*" protobuf = "4.25.1" pydantic = "*" @@ -814,7 +815,7 @@ requests = "*" rich = "*" sentry-sdk = "*" tabulate = "*" -tenacity = ">=8.2.3,<8.3.0" +tenacity = ">=8.4.1,<8.5.0" tqdm = "*" typer = "*" @@ -1397,61 +1398,61 @@ test = ["objgraph", "psutil"] [[package]] name = "grpcio" -version = "1.65.5" +version = "1.63.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.65.5-cp310-cp310-linux_armv7l.whl", hash = "sha256:b67d450f1e008fedcd81e097a3a400a711d8be1a8b20f852a7b8a73fead50fe3"}, - {file = "grpcio-1.65.5-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:a70a20eed87bba647a38bedd93b3ce7db64b3f0e8e0952315237f7f5ca97b02d"}, - {file = "grpcio-1.65.5-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f79c87c114bf37adf408026b9e2e333fe9ff31dfc9648f6f80776c513145c813"}, - {file = "grpcio-1.65.5-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f17f9fa2d947dbfaca01b3ab2c62eefa8240131fdc67b924eb42ce6032e3e5c1"}, - {file = "grpcio-1.65.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32d60e18ff7c34fe3f6db3d35ad5c6dc99f5b43ff3982cb26fad4174462d10b1"}, - {file = "grpcio-1.65.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fe6505376f5b00bb008e4e1418152e3ad3d954b629da286c7913ff3cfc0ff740"}, - {file = "grpcio-1.65.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:33158e56c6378063923c417e9fbdb28660b6e0e2835af42e67f5a7793f587af7"}, - {file = "grpcio-1.65.5-cp310-cp310-win32.whl", hash = "sha256:1cbc208edb9acf1cc339396a1a36b83796939be52f34e591c90292045b579fbf"}, - {file = "grpcio-1.65.5-cp310-cp310-win_amd64.whl", hash = "sha256:bc74f3f745c37e2c5685c9d2a2d5a94de00f286963f5213f763ae137bf4f2358"}, - {file = "grpcio-1.65.5-cp311-cp311-linux_armv7l.whl", hash = "sha256:3207ae60d07e5282c134b6e02f9271a2cb523c6d7a346c6315211fe2bf8d61ed"}, - {file = "grpcio-1.65.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a2f80510f99f82d4eb825849c486df703f50652cea21c189eacc2b84f2bde764"}, - {file = "grpcio-1.65.5-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:a80e9a5e3f93c54f5eb82a3825ea1fc4965b2fa0026db2abfecb139a5c4ecdf1"}, - {file = "grpcio-1.65.5-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b2944390a496567de9e70418f3742b477d85d8ca065afa90432edc91b4bb8ad"}, - {file = "grpcio-1.65.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3655139d7be213c32c79ef6fb2367cae28e56ef68e39b1961c43214b457f257"}, - {file = "grpcio-1.65.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:05f02d68fc720e085f061b704ee653b181e6d5abfe315daef085719728d3d1fd"}, - {file = "grpcio-1.65.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1c4caafe71aef4dabf53274bbf4affd6df651e9f80beedd6b8e08ff438ed3260"}, - {file = "grpcio-1.65.5-cp311-cp311-win32.whl", hash = "sha256:84c901cdec16a092099f251ef3360d15e29ef59772150fa261d94573612539b5"}, - {file = "grpcio-1.65.5-cp311-cp311-win_amd64.whl", hash = "sha256:11f8b16121768c1cb99d7dcb84e01510e60e6a206bf9123e134118802486f035"}, - {file = "grpcio-1.65.5-cp312-cp312-linux_armv7l.whl", hash = "sha256:ee6ed64a27588a2c94e8fa84fe8f3b5c89427d4d69c37690903d428ec61ca7e4"}, - {file = "grpcio-1.65.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:76991b7a6fb98630a3328839755181ce7c1aa2b1842aa085fd4198f0e5198960"}, - {file = "grpcio-1.65.5-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:89c00a18801b1ed9cc441e29b521c354725d4af38c127981f2c950c796a09b6e"}, - {file = "grpcio-1.65.5-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:078038e150a897e5e402ed3d57f1d31ebf604cbed80f595bd281b5da40762a92"}, - {file = "grpcio-1.65.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c97962720489ef31b5ad8a916e22bc31bba3664e063fb9f6702dce056d4aa61b"}, - {file = "grpcio-1.65.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b8270b15b99781461b244f5c81d5c2bc9696ab9189fb5ff86c841417fb3b39fe"}, - {file = "grpcio-1.65.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8e5c4c15ac3fe1eb68e46bc51e66ad29be887479f231f8237cf8416058bf0cc1"}, - {file = "grpcio-1.65.5-cp312-cp312-win32.whl", hash = "sha256:f5b5970341359341d0e4c789da7568264b2a89cd976c05ea476036852b5950cd"}, - {file = "grpcio-1.65.5-cp312-cp312-win_amd64.whl", hash = "sha256:238a625f391a1b9f5f069bdc5930f4fd71b74426bea52196fc7b83f51fa97d34"}, - {file = "grpcio-1.65.5-cp38-cp38-linux_armv7l.whl", hash = "sha256:6c4e62bcf297a1568f627f39576dbfc27f1e5338a691c6dd5dd6b3979da51d1c"}, - {file = "grpcio-1.65.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d7df567b67d16d4177835a68d3f767bbcbad04da9dfb52cbd19171f430c898bd"}, - {file = "grpcio-1.65.5-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:b7ca419f1462390851eec395b2089aad1e49546b52d4e2c972ceb76da69b10f8"}, - {file = "grpcio-1.65.5-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fa36dd8496d3af0d40165252a669fa4f6fd2db4b4026b9a9411cbf060b9d6a15"}, - {file = "grpcio-1.65.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a101696f9ece90a0829988ff72f1b1ea2358f3df035bdf6d675dd8b60c2c0894"}, - {file = "grpcio-1.65.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2a6d8169812932feac514b420daffae8ab8e36f90f3122b94ae767e633296b17"}, - {file = "grpcio-1.65.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:47d0aaaab82823f0aa6adea5184350b46e2252e13a42a942db84da5b733f2e05"}, - {file = "grpcio-1.65.5-cp38-cp38-win32.whl", hash = "sha256:85ae8f8517d5bcc21fb07dbf791e94ed84cc28f84c903cdc2bd7eaeb437c8f45"}, - {file = "grpcio-1.65.5-cp38-cp38-win_amd64.whl", hash = "sha256:770bd4bd721961f6dd8049bc27338564ba8739913f77c0f381a9815e465ff965"}, - {file = "grpcio-1.65.5-cp39-cp39-linux_armv7l.whl", hash = "sha256:ab5ec837d8cee8dbce9ef6386125f119b231e4333cc6b6d57b6c5c7c82a72331"}, - {file = "grpcio-1.65.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cabd706183ee08d8026a015af5819a0b3a8959bdc9d1f6fdacd1810f09200f2a"}, - {file = "grpcio-1.65.5-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:ec71fc5b39821ad7d80db7473c8f8c2910f3382f0ddadfbcfc2c6c437107eb67"}, - {file = "grpcio-1.65.5-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a9e35bcb045e39d7cac30464c285389b9a816ac2067e4884ad2c02e709ef8e"}, - {file = "grpcio-1.65.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d750e9330eb14236ca11b78d0c494eed13d6a95eb55472298f0e547c165ee324"}, - {file = "grpcio-1.65.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2b91ce647b6307f25650872454a4d02a2801f26a475f90d0b91ed8110baae589"}, - {file = "grpcio-1.65.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8da58ff80bc4556cf29bc03f5fff1f03b8387d6aaa7b852af9eb65b2cf833be4"}, - {file = "grpcio-1.65.5-cp39-cp39-win32.whl", hash = "sha256:7a412959aa5f08c5ac04aa7b7c3c041f5e4298cadd4fcc2acff195b56d185ebc"}, - {file = "grpcio-1.65.5-cp39-cp39-win_amd64.whl", hash = "sha256:55714ea852396ec9568f45f487639945ab674de83c12bea19d5ddbc3ae41ada3"}, - {file = "grpcio-1.65.5.tar.gz", hash = "sha256:ec6f219fb5d677a522b0deaf43cea6697b16f338cb68d009e30930c4aa0d2209"}, + {file = "grpcio-1.63.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2e93aca840c29d4ab5db93f94ed0a0ca899e241f2e8aec6334ab3575dc46125c"}, + {file = "grpcio-1.63.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:91b73d3f1340fefa1e1716c8c1ec9930c676d6b10a3513ab6c26004cb02d8b3f"}, + {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:b3afbd9d6827fa6f475a4f91db55e441113f6d3eb9b7ebb8fb806e5bb6d6bd0d"}, + {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f3f6883ce54a7a5f47db43289a0a4c776487912de1a0e2cc83fdaec9685cc9f"}, + {file = "grpcio-1.63.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf8dae9cc0412cb86c8de5a8f3be395c5119a370f3ce2e69c8b7d46bb9872c8d"}, + {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:08e1559fd3b3b4468486b26b0af64a3904a8dbc78d8d936af9c1cf9636eb3e8b"}, + {file = "grpcio-1.63.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5c039ef01516039fa39da8a8a43a95b64e288f79f42a17e6c2904a02a319b357"}, + {file = "grpcio-1.63.0-cp310-cp310-win32.whl", hash = "sha256:ad2ac8903b2eae071055a927ef74121ed52d69468e91d9bcbd028bd0e554be6d"}, + {file = "grpcio-1.63.0-cp310-cp310-win_amd64.whl", hash = "sha256:b2e44f59316716532a993ca2966636df6fbe7be4ab6f099de6815570ebe4383a"}, + {file = "grpcio-1.63.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:f28f8b2db7b86c77916829d64ab21ff49a9d8289ea1564a2b2a3a8ed9ffcccd3"}, + {file = "grpcio-1.63.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:65bf975639a1f93bee63ca60d2e4951f1b543f498d581869922910a476ead2f5"}, + {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:b5194775fec7dc3dbd6a935102bb156cd2c35efe1685b0a46c67b927c74f0cfb"}, + {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4cbb2100ee46d024c45920d16e888ee5d3cf47c66e316210bc236d5bebc42b3"}, + {file = "grpcio-1.63.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ff737cf29b5b801619f10e59b581869e32f400159e8b12d7a97e7e3bdeee6a2"}, + {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cd1e68776262dd44dedd7381b1a0ad09d9930ffb405f737d64f505eb7f77d6c7"}, + {file = "grpcio-1.63.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:93f45f27f516548e23e4ec3fbab21b060416007dbe768a111fc4611464cc773f"}, + {file = "grpcio-1.63.0-cp311-cp311-win32.whl", hash = "sha256:878b1d88d0137df60e6b09b74cdb73db123f9579232c8456f53e9abc4f62eb3c"}, + {file = "grpcio-1.63.0-cp311-cp311-win_amd64.whl", hash = "sha256:756fed02dacd24e8f488f295a913f250b56b98fb793f41d5b2de6c44fb762434"}, + {file = "grpcio-1.63.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:93a46794cc96c3a674cdfb59ef9ce84d46185fe9421baf2268ccb556f8f81f57"}, + {file = "grpcio-1.63.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a7b19dfc74d0be7032ca1eda0ed545e582ee46cd65c162f9e9fc6b26ef827dc6"}, + {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:8064d986d3a64ba21e498b9a376cbc5d6ab2e8ab0e288d39f266f0fca169b90d"}, + {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:219bb1848cd2c90348c79ed0a6b0ea51866bc7e72fa6e205e459fedab5770172"}, + {file = "grpcio-1.63.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a2d60cd1d58817bc5985fae6168d8b5655c4981d448d0f5b6194bbcc038090d2"}, + {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e350cb096e5c67832e9b6e018cf8a0d2a53b2a958f6251615173165269a91b0"}, + {file = "grpcio-1.63.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:56cdf96ff82e3cc90dbe8bac260352993f23e8e256e063c327b6cf9c88daf7a9"}, + {file = "grpcio-1.63.0-cp312-cp312-win32.whl", hash = "sha256:3a6d1f9ea965e750db7b4ee6f9fdef5fdf135abe8a249e75d84b0a3e0c668a1b"}, + {file = "grpcio-1.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:d2497769895bb03efe3187fb1888fc20e98a5f18b3d14b606167dacda5789434"}, + {file = "grpcio-1.63.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:fdf348ae69c6ff484402cfdb14e18c1b0054ac2420079d575c53a60b9b2853ae"}, + {file = "grpcio-1.63.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a3abfe0b0f6798dedd2e9e92e881d9acd0fdb62ae27dcbbfa7654a57e24060c0"}, + {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:6ef0ad92873672a2a3767cb827b64741c363ebaa27e7f21659e4e31f4d750280"}, + {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b416252ac5588d9dfb8a30a191451adbf534e9ce5f56bb02cd193f12d8845b7f"}, + {file = "grpcio-1.63.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3b77eaefc74d7eb861d3ffbdf91b50a1bb1639514ebe764c47773b833fa2d91"}, + {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b005292369d9c1f80bf70c1db1c17c6c342da7576f1c689e8eee4fb0c256af85"}, + {file = "grpcio-1.63.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cdcda1156dcc41e042d1e899ba1f5c2e9f3cd7625b3d6ebfa619806a4c1aadda"}, + {file = "grpcio-1.63.0-cp38-cp38-win32.whl", hash = "sha256:01799e8649f9e94ba7db1aeb3452188048b0019dc37696b0f5ce212c87c560c3"}, + {file = "grpcio-1.63.0-cp38-cp38-win_amd64.whl", hash = "sha256:6a1a3642d76f887aa4009d92f71eb37809abceb3b7b5a1eec9c554a246f20e3a"}, + {file = "grpcio-1.63.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:75f701ff645858a2b16bc8c9fc68af215a8bb2d5a9b647448129de6e85d52bce"}, + {file = "grpcio-1.63.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cacdef0348a08e475a721967f48206a2254a1b26ee7637638d9e081761a5ba86"}, + {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:0697563d1d84d6985e40ec5ec596ff41b52abb3fd91ec240e8cb44a63b895094"}, + {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6426e1fb92d006e47476d42b8f240c1d916a6d4423c5258ccc5b105e43438f61"}, + {file = "grpcio-1.63.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e48cee31bc5f5a31fb2f3b573764bd563aaa5472342860edcc7039525b53e46a"}, + {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:50344663068041b34a992c19c600236e7abb42d6ec32567916b87b4c8b8833b3"}, + {file = "grpcio-1.63.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:259e11932230d70ef24a21b9fb5bb947eb4703f57865a404054400ee92f42f5d"}, + {file = "grpcio-1.63.0-cp39-cp39-win32.whl", hash = "sha256:a44624aad77bf8ca198c55af811fd28f2b3eaf0a50ec5b57b06c034416ef2d0a"}, + {file = "grpcio-1.63.0-cp39-cp39-win_amd64.whl", hash = "sha256:166e5c460e5d7d4656ff9e63b13e1f6029b122104c1633d5f37eaea348d7356d"}, + {file = "grpcio-1.63.0.tar.gz", hash = "sha256:f3023e14805c61bc439fb40ca545ac3d5740ce66120a678a3c6c2c55b70343d1"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.65.5)"] +protobuf = ["grpcio-tools (>=1.63.0)"] [[package]] name = "grpcio-tools" @@ -1760,22 +1761,22 @@ files = [ [[package]] name = "importlib-metadata" -version = "8.0.0" +version = "7.1.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, - {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, + {file = "importlib_metadata-7.1.0-py3-none-any.whl", hash = "sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570"}, + {file = "importlib_metadata-7.1.0.tar.gz", hash = "sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2"}, ] [package.dependencies] zipp = ">=0.5" [package.extras] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] [[package]] name = "iniconfig" @@ -2559,42 +2560,42 @@ datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] [[package]] name = "opentelemetry-api" -version = "1.26.0" +version = "1.25.0" description = "OpenTelemetry Python API" optional = false python-versions = ">=3.8" files = [ - {file = "opentelemetry_api-1.26.0-py3-none-any.whl", hash = "sha256:7d7ea33adf2ceda2dd680b18b1677e4152000b37ca76e679da71ff103b943064"}, - {file = "opentelemetry_api-1.26.0.tar.gz", hash = "sha256:2bd639e4bed5b18486fef0b5a520aaffde5a18fc225e808a1ac4df363f43a1ce"}, + {file = "opentelemetry_api-1.25.0-py3-none-any.whl", hash = "sha256:757fa1aa020a0f8fa139f8959e53dec2051cc26b832e76fa839a6d76ecefd737"}, + {file = "opentelemetry_api-1.25.0.tar.gz", hash = "sha256:77c4985f62f2614e42ce77ee4c9da5fa5f0bc1e1821085e9a47533a9323ae869"}, ] [package.dependencies] deprecated = ">=1.2.6" -importlib-metadata = ">=6.0,<=8.0.0" +importlib-metadata = ">=6.0,<=7.1" [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.26.0" +version = "1.25.0" description = "OpenTelemetry Protobuf encoding" optional = false python-versions = ">=3.8" files = [ - {file = "opentelemetry_exporter_otlp_proto_common-1.26.0-py3-none-any.whl", hash = "sha256:ee4d8f8891a1b9c372abf8d109409e5b81947cf66423fd998e56880057afbc71"}, - {file = "opentelemetry_exporter_otlp_proto_common-1.26.0.tar.gz", hash = "sha256:bdbe50e2e22a1c71acaa0c8ba6efaadd58882e5a5978737a44a4c4b10d304c92"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.25.0-py3-none-any.whl", hash = "sha256:15637b7d580c2675f70246563363775b4e6de947871e01d0f4e3881d1848d693"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.25.0.tar.gz", hash = "sha256:c93f4e30da4eee02bacd1e004eb82ce4da143a2f8e15b987a9f603e0a85407d3"}, ] [package.dependencies] -opentelemetry-proto = "1.26.0" +opentelemetry-proto = "1.25.0" [[package]] name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.26.0" +version = "1.25.0" description = "OpenTelemetry Collector Protobuf over gRPC Exporter" optional = false python-versions = ">=3.8" files = [ - {file = "opentelemetry_exporter_otlp_proto_grpc-1.26.0-py3-none-any.whl", hash = "sha256:e2be5eff72ebcb010675b818e8d7c2e7d61ec451755b8de67a140bc49b9b0280"}, - {file = "opentelemetry_exporter_otlp_proto_grpc-1.26.0.tar.gz", hash = "sha256:a65b67a9a6b06ba1ec406114568e21afe88c1cdb29c464f2507d529eb906d8ae"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.25.0-py3-none-any.whl", hash = "sha256:3131028f0c0a155a64c430ca600fd658e8e37043cb13209f0109db5c1a3e4eb4"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.25.0.tar.gz", hash = "sha256:c0b1661415acec5af87625587efa1ccab68b873745ca0ee96b69bb1042087eac"}, ] [package.dependencies] @@ -2602,19 +2603,19 @@ deprecated = ">=1.2.6" googleapis-common-protos = ">=1.52,<2.0" grpcio = ">=1.0.0,<2.0.0" opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.26.0" -opentelemetry-proto = "1.26.0" -opentelemetry-sdk = ">=1.26.0,<1.27.0" +opentelemetry-exporter-otlp-proto-common = "1.25.0" +opentelemetry-proto = "1.25.0" +opentelemetry-sdk = ">=1.25.0,<1.26.0" [[package]] name = "opentelemetry-proto" -version = "1.26.0" +version = "1.25.0" description = "OpenTelemetry Python Proto" optional = false python-versions = ">=3.8" files = [ - {file = "opentelemetry_proto-1.26.0-py3-none-any.whl", hash = "sha256:6c4d7b4d4d9c88543bcf8c28ae3f8f0448a753dc291c18c5390444c90b76a725"}, - {file = "opentelemetry_proto-1.26.0.tar.gz", hash = "sha256:c5c18796c0cab3751fc3b98dee53855835e90c0422924b484432ac852d93dc1e"}, + {file = "opentelemetry_proto-1.25.0-py3-none-any.whl", hash = "sha256:f07e3341c78d835d9b86665903b199893befa5e98866f63d22b00d0b7ca4972f"}, + {file = "opentelemetry_proto-1.25.0.tar.gz", hash = "sha256:35b6ef9dc4a9f7853ecc5006738ad40443701e52c26099e197895cbda8b815a3"}, ] [package.dependencies] @@ -2622,34 +2623,33 @@ protobuf = ">=3.19,<5.0" [[package]] name = "opentelemetry-sdk" -version = "1.26.0" +version = "1.25.0" description = "OpenTelemetry Python SDK" optional = false python-versions = ">=3.8" files = [ - {file = "opentelemetry_sdk-1.26.0-py3-none-any.whl", hash = "sha256:feb5056a84a88670c041ea0ded9921fca559efec03905dddeb3885525e0af897"}, - {file = "opentelemetry_sdk-1.26.0.tar.gz", hash = "sha256:c90d2868f8805619535c05562d699e2f4fb1f00dbd55a86dcefca4da6fa02f85"}, + {file = "opentelemetry_sdk-1.25.0-py3-none-any.whl", hash = "sha256:d97ff7ec4b351692e9d5a15af570c693b8715ad78b8aafbec5c7100fe966b4c9"}, + {file = "opentelemetry_sdk-1.25.0.tar.gz", hash = "sha256:ce7fc319c57707ef5bf8b74fb9f8ebdb8bfafbe11898410e0d2a761d08a98ec7"}, ] [package.dependencies] -opentelemetry-api = "1.26.0" -opentelemetry-semantic-conventions = "0.47b0" +opentelemetry-api = "1.25.0" +opentelemetry-semantic-conventions = "0.46b0" typing-extensions = ">=3.7.4" [[package]] name = "opentelemetry-semantic-conventions" -version = "0.47b0" +version = "0.46b0" description = "OpenTelemetry Semantic Conventions" optional = false python-versions = ">=3.8" files = [ - {file = "opentelemetry_semantic_conventions-0.47b0-py3-none-any.whl", hash = "sha256:4ff9d595b85a59c1c1413f02bba320ce7ea6bf9e2ead2b0913c4395c7bbc1063"}, - {file = "opentelemetry_semantic_conventions-0.47b0.tar.gz", hash = "sha256:a8d57999bbe3495ffd4d510de26a97dadc1dace53e0275001b2c1b2f67992a7e"}, + {file = "opentelemetry_semantic_conventions-0.46b0-py3-none-any.whl", hash = "sha256:6daef4ef9fa51d51855d9f8e0ccd3a1bd59e0e545abe99ac6203804e36ab3e07"}, + {file = "opentelemetry_semantic_conventions-0.46b0.tar.gz", hash = "sha256:fbc982ecbb6a6e90869b15c1673be90bd18c8a56ff1cffc0864e38e2edffaefa"}, ] [package.dependencies] -deprecated = ">=1.2.6" -opentelemetry-api = "1.26.0" +opentelemetry-api = "1.25.0" [[package]] name = "orjson" @@ -4327,17 +4327,18 @@ widechars = ["wcwidth"] [[package]] name = "tenacity" -version = "8.2.3" +version = "8.4.2" description = "Retry code until it succeeds" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tenacity-8.2.3-py3-none-any.whl", hash = "sha256:ce510e327a630c9e1beaf17d42e6ffacc88185044ad85cf74c0a8887c6a0f88c"}, - {file = "tenacity-8.2.3.tar.gz", hash = "sha256:5398ef0d78e63f40007c1fb4c0bff96e1911394d2fa8d194f77619c05ff6cc8a"}, + {file = "tenacity-8.4.2-py3-none-any.whl", hash = "sha256:9e6f7cf7da729125c7437222f8a522279751cdfbe6b67bfe64f75d3a348661b2"}, + {file = "tenacity-8.4.2.tar.gz", hash = "sha256:cd80a53a79336edba8489e767f729e4f391c896956b57140b5d7511a64bbd3ef"}, ] [package.extras] -doc = ["reno", "sphinx", "tornado (>=4.5)"] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] name = "tiktoken" @@ -5252,4 +5253,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = ">=3.12.*, <4.0" -content-hash = "e4ff011773d68c52f6c229b66655168df1b14fb0faaa9f3aed9141178bae79d1" +content-hash = "d5bdcee844dbb90791d8b405e91bee2194117d8d7cac170e33395ce656900cff" diff --git a/wren-ai-service/pyproject.toml b/wren-ai-service/pyproject.toml index 3a8b5ec5c..06bc20251 100644 --- a/wren-ai-service/pyproject.toml +++ b/wren-ai-service/pyproject.toml @@ -35,7 +35,7 @@ watchdog = "==4.0.0" [tool.poetry.group.eval.dependencies] tomlkit = "==0.13.0" -deepeval = "==0.21.68" +deepeval = "==1.0.6" streamlit-tags = "==1.2.8" gitpython = "==3.1.43" pandas = "==2.2.2"