mirror of
https://github.com/Canner/WrenAI.git
synced 2026-10-30 18:22:21 +08:00
Add streamlit app for eval analysis (#12)
* resolve conflict * add missed files * add streamlit for ask pipeline eval comparison * fix typo * refine ui for prediction results comparison * fix formatting * rename vulcansql-core-server to wren-engine and add missing .env
This commit is contained in:
Chih-Yu Yeh
+12
-17
@@ -1,21 +1,16 @@
|
||||
# vectorstore
|
||||
vectorstore
|
||||
retrieve_db/
|
||||
wren-ai-service/qdrant_storage/
|
||||
|
||||
# app
|
||||
.env
|
||||
.env.*
|
||||
!.env.*.example
|
||||
!src/eval/vulcansql-core-server/.env
|
||||
src/eval/vulcansql-core-server/config.properties
|
||||
outputs/
|
||||
spider/
|
||||
wren-ai-service/**/data/
|
||||
# wren-ai-service
|
||||
wren-ai-service/.env
|
||||
wren-ai-service/.env.*
|
||||
!wren-ai-service/.env.example
|
||||
!wren-ai-service/.env.*.example
|
||||
!wren-ai-service/src/eval/wren-engine/.env
|
||||
wren-ai-service/src/eval/wren-engine/config.properties
|
||||
wren-ai-service/outputs/
|
||||
wren-ai-service/spider/
|
||||
!wren-ai-service/tests/data
|
||||
!src/eval/data
|
||||
poetry.lock
|
||||
assertion.log
|
||||
!wren-ai-service/src/eval/data
|
||||
wren-ai-service/poetry.lock
|
||||
wren-ai-service/assertion.log
|
||||
|
||||
# cache
|
||||
__pycache__
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ENV=dev
|
||||
@@ -1,9 +1,9 @@
|
||||
# docker related
|
||||
# This will determine the prefix of container name.
|
||||
# If the service name in docker-compose.yaml is copilot,
|
||||
# then the container name will be vulcansql-copilot-1.
|
||||
# If the service name in docker-compose.yaml is ai-service,
|
||||
# then the container name will be wren-ai-service-1.
|
||||
# If COMPOSE_PROJECT_NAME is not set, the default prefix name is docker.
|
||||
COMPOSE_PROJECT_NAME=vulcansql
|
||||
COMPOSE_PROJECT_NAME=wren
|
||||
|
||||
# fastapi related
|
||||
UVICORN_PORT=8080
|
||||
|
||||
@@ -24,22 +24,25 @@ stop-qdrant:
|
||||
docker stop qdrant && docker rm qdrant
|
||||
|
||||
# evaluation related
|
||||
streamlit:
|
||||
poetry run streamlit run src/eval/streamlit_app.py
|
||||
|
||||
eval:
|
||||
poetry run python -m src.eval.ask $(args)
|
||||
|
||||
run-vulcansql-core-server:
|
||||
docker compose -f ./src/eval/vulcansql-core-server/docker-compose.yml --env-file .env.dev --env-file ./src/eval/vulcansql-core-server/.env up -d
|
||||
run-wren-engine:
|
||||
docker compose -f ./src/eval/wren-engine/docker-compose.yml --env-file .env.dev --env-file ./src/eval/wren-engine/.env up -d
|
||||
|
||||
stop-vulcansql-core-server:
|
||||
docker compose -f ./src/eval/vulcansql-core-server/docker-compose.yml --env-file .env.dev --env-file ./src/eval/vulcansql-core-server/.env down
|
||||
stop-wren-engine:
|
||||
docker compose -f ./src/eval/wren-engine/docker-compose.yml --env-file .env.dev --env-file ./src/eval/wren-engine/.env down
|
||||
|
||||
psql:
|
||||
docker exec -it vulcansql-engine-1 bash launch-cli.sh
|
||||
docker exec -it wren-engine-1 bash launch-cli.sh
|
||||
|
||||
run-all:
|
||||
make run-qdrant && \
|
||||
make run-vulcansql-core-server
|
||||
make run-wren-engine
|
||||
|
||||
stop-all:
|
||||
make stop-qdrant && \
|
||||
make stop-vulcansql-core-server
|
||||
make stop-wren-engine
|
||||
@@ -11,8 +11,7 @@
|
||||
|
||||
## Start the service for development
|
||||
|
||||
- execute `make start` to start the service
|
||||
- go to `http://UVICORN_HOST:UVICORN_PORT/docs` to see the API documentation and try the API
|
||||
- execute `make start` to start the service and go to `http://UVICORN_HOST:UVICORN_PORT/docs` to see the API documentation and try the API
|
||||
|
||||
## Production Environment Setup
|
||||
|
||||
@@ -23,8 +22,9 @@
|
||||
|
||||
## Pipeline Evaluation(for development)
|
||||
|
||||
- fill in environment variables: `.env.dev` in the src folder and `config.properties` in the src/eval/vulcansql-core-server folder
|
||||
- fill in environment variables: `.env.dev` in the src folder and `config.properties` in the src/eval/wren-engine folder
|
||||
- start docker
|
||||
- run qdrant and vulcansql-core-server docker containers: `make run-all`
|
||||
- run qdrant and wren-engine docker containers: `make run-all`
|
||||
- evaluation: `make eval` and check out the outputs folder
|
||||
- `make streamlit` to compare between the evaluation results
|
||||
- to run individual pipeline: `poetry run python -m src.pipelines.ask.[pipeline_name]` (e.g. `poetry run python -m src.pipelines.ask.retrieval_pipeline`)
|
||||
@@ -1,8 +1,8 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
copilot:
|
||||
image: vulcansql-copilot:latest
|
||||
ai-service:
|
||||
image: wren-ai-service:latest
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile
|
||||
|
||||
@@ -19,6 +19,7 @@ qdrant-haystack = "^3.0.0"
|
||||
backoff = "^2.2.1"
|
||||
ragas-haystack = "^0.1.1"
|
||||
tqdm = "^4.66.2"
|
||||
numpy = "^1.26.4"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
pytest = "^8.0.0"
|
||||
@@ -27,6 +28,8 @@ pytest-cov = "^4.1.0"
|
||||
gdown = "^5.1.0"
|
||||
sqlglot = "^22.1.0"
|
||||
httpx = "^0.27.0"
|
||||
streamlit = "^1.32.1"
|
||||
watchdog = "^4.0.0"
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
|
||||
@@ -13,7 +13,7 @@ from tqdm import tqdm
|
||||
from src.pipelines.ask.components.document_store import init_document_store
|
||||
from src.pipelines.ask.components.embedder import init_embedder
|
||||
from src.pipelines.ask.components.generator import init_generator
|
||||
from src.pipelines.ask.components.prompts import user_prompt_builder
|
||||
from src.pipelines.ask.components.prompts import init_generation_prompt_builder
|
||||
from src.pipelines.ask.components.retriever import init_retriever
|
||||
from src.pipelines.ask.generation_pipeline import Generation
|
||||
from src.pipelines.ask.indexing_pipeline import Indexing
|
||||
@@ -162,7 +162,7 @@ if __name__ == "__main__":
|
||||
generation_pipeline = Generation(
|
||||
generator=generator,
|
||||
with_trace=with_trace,
|
||||
prompt_builder=user_prompt_builder,
|
||||
prompt_builder=init_generation_prompt_builder(),
|
||||
)
|
||||
|
||||
start = time.time()
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
{"question": "what is the full name and id of the college with the largest number of baseball players?", "answer": "SELECT T1.name_full, T1.college_id FROM college AS T1 JOIN player_college AS T2 ON T1.college_id = T2.college_id GROUP BY T1.college_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "Find the full name and id of the college that has the most baseball players.", "answer": "SELECT T1.name_full, T1.college_id FROM college AS T1 JOIN player_college AS T2 ON T1.college_id = T2.college_id GROUP BY T1.college_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "What is average salary of the players in the team named 'Boston Red Stockings' ?", "answer": "SELECT AVG(T1.player_salary) FROM salary AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings'"}
|
||||
{"question": "Compute the average salary of the players in the team called 'Boston Red Stockings'.", "answer": "SELECT AVG(T1.salary) FROM salary AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings'"}
|
||||
{"question": "What are first and last names of players participating in all star game in 1998?", "answer": "SELECT name_first, name_last FROM player AS T1 JOIN all_star AS T2 ON T1.player_id = T2.player_id WHERE YEAR = 1998"}
|
||||
{"question": "List the first and last name for players who participated in all star game in 1998.", "answer": "SELECT name_first, name_last FROM player AS T1 JOIN all_star AS T2 ON T1.player_id = T2.player_id WHERE YEAR = 1998"}
|
||||
{"question": "What are the first name, last name and id of the player with the most all star game experiences? Also list the count.", "answer": "SELECT T1.name_first, T1.name_last, T1.player_id, COUNT(*) FROM player AS T1 JOIN all_star AS T2 ON T1.player_id = T2.player_id GROUP BY T1.player_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "Which player has the most all star game experiences? Give me the first name, last name and id of the player, as well as the number of times the player participated in all star game.", "answer": "SELECT T1.name_first, T1.name_last, T1.player_id, COUNT(*) FROM player AS T1 JOIN all_star AS T2 ON T1.player_id = T2.player_id GROUP BY T1.player_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "How many players enter hall of fame each year?", "answer": "SELECT yearid, COUNT(*) FROM hall_of_fame GROUP BY yearid"}
|
||||
{"question": "Count the number of players who enter hall of fame for each year.", "answer": "SELECT yearid, COUNT(*) FROM hall_of_fame GROUP BY yearid"}
|
||||
{"question": "What is the average number of attendance at home games for each year?", "answer": "SELECT YEAR, AVG(attendance) FROM home_game GROUP BY YEAR"}
|
||||
{"question": "For each year, return the year and the average number of attendance at home games.", "answer": "SELECT YEAR, AVG(attendance) FROM home_game GROUP BY YEAR"}
|
||||
{"question": "In 2014, what are the id and rank of the team that has the largest average number of attendance?", "answer": "SELECT T2.team_id, T2.rank FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id WHERE T1.year = 2014 GROUP BY T1.team_id ORDER BY AVG(T1.attendance) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "Find the id and rank of the team that has the highest average attendance rate in 2014.", "answer": "SELECT T2.team_id, T2.rank FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id WHERE T1.year = 2014 GROUP BY T1.team_id ORDER BY AVG(T1.attendance) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "What are the manager's first name, last name and id who won the most manager award?", "answer": "SELECT T1.name_first, T1.name_last, T2.player_id FROM player AS T1 JOIN manager_award AS T2 ON T1.player_id = T2.player_id GROUP BY T2.player_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "Which manager won the most manager award? Give me the manager's first name, last name and id.", "answer": "SELECT T1.name_first, T1.name_last, T2.player_id FROM player AS T1 JOIN manager_award AS T2 ON T1.player_id = T2.player_id GROUP BY T2.player_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "How many parks are there in the state of NY?", "answer": "SELECT COUNT(*) FROM park WHERE state = 'NY'"}
|
||||
{"question": "Show me the number of parks the state of NY has.", "answer": "SELECT COUNT(*) FROM park WHERE state = 'NY'"}
|
||||
{"question": "Which 3 players won the most player awards? List their full name and id.", "answer": "SELECT T1.name_first, T1.name_last, T1.player_id FROM player AS T1 JOIN player_award AS T2 ON T1.player_id = T2.player_id GROUP BY T1.player_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 3"}
|
||||
{"question": "Find the first name, last name and id for the top three players won the most player awards.", "answer": "SELECT T1.name_first, T1.name_last, T1.player_id FROM player AS T1 JOIN player_award AS T2 ON T1.player_id = T2.player_id GROUP BY T1.player_id ORDER BY COUNT(*) DESC NULLS LAST LIMIT 3"}
|
||||
{"question": "List three countries which are the origins of the least players.", "answer": "SELECT birth_country FROM player GROUP BY birth_country ORDER BY COUNT(*) ASC NULLS FIRST LIMIT 3"}
|
||||
{"question": "What are the three countries that the least players are from?", "answer": "SELECT birth_country FROM player GROUP BY birth_country ORDER BY COUNT(*) ASC NULLS FIRST LIMIT 3"}
|
||||
{"question": "Find all the players' first name and last name who have empty death record.", "answer": "SELECT name_first, name_last FROM player WHERE death_year = ''"}
|
||||
{"question": "What are the first name and last name of the players whose death record is empty?", "answer": "SELECT name_first, name_last FROM player WHERE death_year = ''"}
|
||||
{"question": "How many players born in USA are right-handed batters? That is, have the batter value 'R'.", "answer": "SELECT COUNT(*) FROM player WHERE birth_country = 'USA' AND bats = 'R'"}
|
||||
{"question": "Count the number of players who were born in USA and have bats information 'R'.", "answer": "SELECT COUNT(*) FROM player WHERE birth_country = 'USA' AND bats = 'R'"}
|
||||
{"question": "What is the average height of the players from the college named 'Yale University'?", "answer": "SELECT AVG(T1.height) FROM player AS T1 JOIN player_college AS T2 ON T1.player_id = T2.player_id JOIN college AS T3 ON T3.college_id = T2.college_id WHERE T3.name_full = 'Yale University'"}
|
||||
{"question": "Find the average height of the players who belong to the college called 'Yale University'.", "answer": "SELECT AVG(T1.height) FROM player AS T1 JOIN player_college AS T2 ON T1.player_id = T2.player_id JOIN college AS T3 ON T3.college_id = T2.college_id WHERE T3.name_full = 'Yale University'"}
|
||||
{"question": "What is the highest salary among each team? List the team name, id and maximum salary.", "answer": "SELECT T1.name, T1.team_id, MAX(T2.player_salary) FROM team AS T1 JOIN salary AS T2 ON T1.team_id = T2.team_id GROUP BY T1.team_id"}
|
||||
{"question": "For each team, return the team name, id and the maximum salary among the team.", "answer": "SELECT T1.name, T1.team_id, MAX(T2.player_salary) FROM team AS T1 JOIN salary AS T2 ON T1.team_id = T2.team_id GROUP BY T1.team_id"}
|
||||
{"question": "What are the name and id of the team offering the lowest average salary?", "answer": "SELECT T1.name, T1.team_id FROM team AS T1 JOIN salary AS T2 ON T1.team_id = T2.team_id GROUP BY T1.team_id ORDER BY AVG(T2.salary) ASC NULLS FIRST LIMIT 1"}
|
||||
{"question": "Which team offers the lowest average salary? Give me the name and id of the team.", "answer": "SELECT T1.name, T1.team_id FROM team AS T1 JOIN salary AS T2 ON T1.team_id = T2.team_id GROUP BY T1.team_id ORDER BY AVG(T2.salary) ASC NULLS FIRST LIMIT 1"}
|
||||
{"question": "Find the players' first name and last name who won award both in 1960 and in 1961.", "answer": "SELECT T1.name_first, T1.name_last FROM player AS T1, player_award AS T2 WHERE T2.year = 1960 INTERSECT SELECT T1.name_first, T1.name_last FROM player AS T1, player_award AS T2 WHERE T2.year = 1961"}
|
||||
{"question": "Which players won awards in both 1960 and 1961? Return their first names and last names.", "answer": "SELECT T1.name_first, T1.name_last FROM player AS T1, player_award AS T2 WHERE T2.year = 1960 INTERSECT SELECT T1.name_first, T1.name_last FROM player AS T1, player_award AS T2 WHERE T2.year = 1961"}
|
||||
{"question": "List players' first name and last name who have weight greater than 220 or height shorter than 75.", "answer": "SELECT name_first, name_last FROM player WHERE weight > 220 OR height < 75"}
|
||||
{"question": "What are the first name and last name of the players who have weight above 220 or height below 75?", "answer": "SELECT name_first, name_last FROM player WHERE weight > 220 OR height < 75"}
|
||||
{"question": "List the maximum scores of the team Boston Red Stockings when the team won in postseason?", "answer": "SELECT MAX(T1.wins) FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T2.name = 'Boston Red Stockings'"}
|
||||
{"question": "What are the maximum scores the team Boston Red Stockings got when the team won in postseason?", "answer": "SELECT MAX(T1.wins) FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T2.name = 'Boston Red Stockings'"}
|
||||
{"question": "How many times did Boston Red Stockings lose in 2009 postseason?", "answer": "SELECT COUNT(*) FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_loser = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year = 2009"}
|
||||
{"question": "Count the number of times the team \"Boston Red Stockings\" lost in 2009 postseason.", "answer": "SELECT COUNT(*) FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_loser = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year = 2009"}
|
||||
{"question": "What are the name and id of the team with the most victories in 2008 postseason?", "answer": "SELECT T2.name, T1.team_id_winner FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T1.year = 2008 GROUP BY T1.team_id_winner ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "Find the name and id of the team that won the most times in 2008 postseason.", "answer": "SELECT T2.name, T1.team_id_winner FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T1.year = 2008 GROUP BY T1.team_id_winner ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "What is the number of wins the team Boston Red Stockings got in the postseasons each year in history?", "answer": "SELECT COUNT(*), T1.year FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' GROUP BY T1.year"}
|
||||
{"question": "For each year, return the year and the number of times the team Boston Red Stockings won in the postseasons.", "answer": "SELECT COUNT(*), T1.year FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' GROUP BY T1.year"}
|
||||
{"question": "What is the total number of postseason games that team Boston Red Stockings participated in?", "answer": "SELECT COUNT(*) FROM (SELECT * FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' UNION SELECT * FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_loser = T2.team_id_br WHERE T2.name = 'Boston Red Stockings')"}
|
||||
{"question": "How many times in total did the team Boston Red Stockings participate in postseason games?", "answer": "SELECT COUNT(*) FROM (SELECT * FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_winner = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' UNION SELECT * FROM postseason AS T1 JOIN team AS T2 ON T1.team_id_loser = T2.team_id_br WHERE T2.name = 'Boston Red Stockings')"}
|
||||
{"question": "How many games in 1885 postseason resulted in ties (that is, the value of \"ties\" is '1')?", "answer": "SELECT COUNT(*) FROM postseason WHERE YEAR = 1885 AND ties = 1"}
|
||||
{"question": "Find the number of tied games (the value of \"ties\" is '1') in 1885 postseason.", "answer": "SELECT COUNT(*) FROM postseason WHERE YEAR = 1885 AND ties = 1"}
|
||||
{"question": "What is the total salary paid by team Boston Red Stockings in 2010?", "answer": "SELECT SUM(T1.player_salary) FROM salary AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year = 2010"}
|
||||
{"question": "What is the total salary expenses of team Boston Red Stockings in 2010?", "answer": "SELECT SUM(T1.player_salary) FROM salary AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year = 2010"}
|
||||
{"question": "How many players were in the team Boston Red Stockings in 2000?", "answer": "SELECT COUNT(*) FROM salary AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year = 2000"}
|
||||
{"question": "How many players did Boston Red Stockings have in 2000?", "answer": "SELECT COUNT(*) FROM salary AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year = 2000"}
|
||||
{"question": "List the 3 highest salaries of the players in 2001?", "answer": "SELECT player_salary FROM salary WHERE YEAR = 2001 ORDER BY player_salary DESC NULLS LAST LIMIT 3"}
|
||||
{"question": "How much salary did the top 3 well-paid players get in 2001?", "answer": "SELECT player_salary FROM salary WHERE YEAR = 2001 ORDER BY player_salary DESC NULLS LAST LIMIT 3"}
|
||||
{"question": "What were all the salary values of players in 2010 and 2001?", "answer": "SELECT player_salary FROM salary WHERE YEAR = 2010 UNION SELECT player_salary FROM salary WHERE YEAR = 2001"}
|
||||
{"question": "List all the salary values players received in 2010 and 2001.", "answer": "SELECT player_salary FROM salary WHERE YEAR = 2010 UNION SELECT player_salary FROM salary WHERE YEAR = 2001"}
|
||||
{"question": "In which year did the least people enter hall of fame?", "answer": "SELECT yearid FROM hall_of_fame GROUP BY yearid ORDER BY COUNT(*) ASC NULLS FIRST LIMIT 1"}
|
||||
{"question": "Find the year in which the least people enter hall of fame.", "answer": "SELECT yearid FROM hall_of_fame GROUP BY yearid ORDER BY COUNT(*) ASC NULLS FIRST LIMIT 1"}
|
||||
{"question": "How many parks are there in Atlanta city?", "answer": "SELECT COUNT(*) FROM park WHERE city = 'Atlanta'"}
|
||||
{"question": "How many parks does Atlanta city have?", "answer": "SELECT COUNT(*) FROM park WHERE city = 'Atlanta'"}
|
||||
{"question": "How many games were played in park \"Columbia Park\" in 1907?", "answer": "SELECT COUNT(*) FROM home_game AS T1 JOIN park AS T2 ON T1.park_id = T2.park_id WHERE T1.year = 1907 AND T2.park_name = 'Columbia Park'"}
|
||||
{"question": "Count the number of games taken place in park \"Columbia Park\" in 1907.", "answer": "SELECT COUNT(*) FROM home_game AS T1 JOIN park AS T2 ON T1.park_id = T2.park_id WHERE T1.year = 1907 AND T2.park_name = 'Columbia Park'"}
|
||||
{"question": "How many games were played in city Atlanta in 2000?", "answer": "SELECT COUNT(*) FROM home_game AS T1 JOIN park AS T2 ON T1.park_id = T2.park_id WHERE T1.year = 2000 AND T2.city = 'Atlanta'"}
|
||||
{"question": "Find the number of games taken place in city Atlanta in 2000.", "answer": "SELECT COUNT(*) FROM home_game AS T1 JOIN park AS T2 ON T1.park_id = T2.park_id WHERE T1.year = 2000 AND T2.city = 'Atlanta'"}
|
||||
{"question": "What is the total home game attendance of team Boston Red Stockings from 2000 to 2010?", "answer": "SELECT SUM(T1.attendance) FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year BETWEEN 2000 AND 2010"}
|
||||
{"question": "How many games in total did team Boston Red Stockings attend from 2000 to 2010?", "answer": "SELECT SUM(T1.attendance) FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year BETWEEN 2000 AND 2010"}
|
||||
{"question": "How much did the the player with first name Len and last name Barker earn between 1985 to 1990 in total?", "answer": "SELECT SUM(T1.player_salary) FROM salary AS T1 JOIN player AS T2 ON T1.player_id = T2.player_id WHERE T2.name_first = 'Len' AND T2.name_last = 'Barker' AND T1.year BETWEEN 1985 AND 1990"}
|
||||
{"question": "Compute the total salary that the player with first name Len and last name Barker received between 1985 to 1990.", "answer": "SELECT SUM(T1.player_salary) FROM salary AS T1 JOIN player AS T2 ON T1.player_id = T2.player_id WHERE T2.name_first = 'Len' AND T2.name_last = 'Barker' AND T1.year BETWEEN 1985 AND 1990"}
|
||||
{"question": "List players' first name and last name who received salary from team Washington Nationals in both 2005 and 2007.", "answer": "SELECT T2.name_first, T2.name_last FROM salary AS T1 JOIN player AS T2 ON T1.player_id = T2.player_id JOIN team AS T3 ON T3.team_id_br = T1.team_id WHERE T1.year = 2005 AND T3.name = 'Washington Nationals' INTERSECT SELECT T2.name_first, T2.name_last FROM salary AS T1 JOIN player AS T2 ON T1.player_id = T2.player_id JOIN team AS T3 ON T3.team_id_br = T1.team_id WHERE T1.year = 2007 AND T3.name = 'Washington Nationals'"}
|
||||
{"question": "What are the first name and last name of the players who were paid salary by team Washington Nationals in both 2005 and 2007?", "answer": "SELECT T2.name_first, T2.name_last FROM salary AS T1 JOIN player AS T2 ON T1.player_id = T2.player_id JOIN team AS T3 ON T3.team_id_br = T1.team_id WHERE T1.year = 2005 AND T3.name = 'Washington Nationals' INTERSECT SELECT T2.name_first, T2.name_last FROM salary AS T1 JOIN player AS T2 ON T1.player_id = T2.player_id JOIN team AS T3 ON T3.team_id_br = T1.team_id WHERE T1.year = 2007 AND T3.name = 'Washington Nationals'"}
|
||||
{"question": "How many home games did the team Boston Red Stockings play from 1990 to 2000 in total?", "answer": "SELECT SUM(T1.games) FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year BETWEEN 1990 AND 2000"}
|
||||
{"question": "Count the total number of games the team Boston Red Stockings attended from 1990 to 2000.", "answer": "SELECT SUM(T1.games) FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T2.name = 'Boston Red Stockings' AND T1.year BETWEEN 1990 AND 2000"}
|
||||
{"question": "Which team had the least number of attendances in home games in 1980?", "answer": "SELECT T2.name FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T1.year = 1980 ORDER BY T1.attendance ASC NULLS FIRST LIMIT 1"}
|
||||
{"question": "Find the team that attended the least number of home games in 1980.", "answer": "SELECT T2.name FROM home_game AS T1 JOIN team AS T2 ON T1.team_id = T2.team_id_br WHERE T1.year = 1980 ORDER BY T1.attendance ASC NULLS FIRST LIMIT 1"}
|
||||
{"question": "List the names of states that have more than 2 parks.", "answer": "SELECT state FROM park GROUP BY state HAVING COUNT(*) > 2"}
|
||||
{"question": "Which states have more than 2 parks?", "answer": "SELECT state FROM park GROUP BY state HAVING COUNT(*) > 2"}
|
||||
{"question": "How many team franchises are active, with active value 'Y'?", "answer": "SELECT COUNT(*) FROM team_franchise WHERE active = 'Y'"}
|
||||
{"question": "Find the number of team franchises that are active (have 'Y' as \"active\" information).", "answer": "SELECT COUNT(*) FROM team_franchise WHERE active = 'Y'"}
|
||||
{"question": "Which cities have 2 to 4 parks?", "answer": "SELECT city FROM park GROUP BY city HAVING COUNT(*) BETWEEN 2 AND 4"}
|
||||
{"question": "Find all the cities that have 2 to 4 parks.", "answer": "SELECT city FROM park GROUP BY city HAVING COUNT(*) BETWEEN 2 AND 4"}
|
||||
{"question": "Which park had most attendances in 2008?", "answer": "SELECT T2.park_name FROM home_game AS T1 JOIN park AS T2 ON T1.park_id = T2.park_id WHERE T1.year = 2008 ORDER BY T1.attendance DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "Which park did the most people attend in 2008?", "answer": "SELECT T2.park_name FROM home_game AS T1 JOIN park AS T2 ON T1.park_id = T2.park_id WHERE T1.year = 2008 ORDER BY T1.attendance DESC NULLS LAST LIMIT 1"}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
||||
{"question": "How many books are there?", "answer": "SELECT COUNT(*) FROM book"}
|
||||
{"question": "List the writers of the books in ascending alphabetical order.", "answer": "SELECT Writer FROM book ORDER BY Writer ASC NULLS FIRST"}
|
||||
{"question": "List the titles of the books in ascending order of issues.", "answer": "SELECT Title FROM book ORDER BY Issues ASC NULLS FIRST"}
|
||||
{"question": "What are the titles of the books whose writer is not \"Elaine Lee\"?", "answer": "SELECT Title FROM book WHERE Writer <> \"Elaine Lee\""}
|
||||
{"question": "What are the title and issues of the books?", "answer": "SELECT Title, Issues FROM book"}
|
||||
{"question": "What are the dates of publications in descending order of price?", "answer": "SELECT Publication_Date FROM publication ORDER BY Price DESC NULLS LAST"}
|
||||
{"question": "What are the distinct publishers of publications with price higher than 5000000?", "answer": "SELECT DISTINCT Publisher FROM publication WHERE Price > 5000000"}
|
||||
{"question": "List the publisher of the publication with the highest price.", "answer": "SELECT Publisher FROM publication ORDER BY Price DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "List the publication dates of publications with 3 lowest prices.", "answer": "SELECT Publication_Date FROM publication ORDER BY Price ASC NULLS FIRST LIMIT 3"}
|
||||
{"question": "Show the title and publication dates of books.", "answer": "SELECT T1.Title, T2.Publication_Date FROM book AS T1 JOIN publication AS T2 ON T1.Book_ID = T2.Book_ID"}
|
||||
{"question": "Show writers who have published a book with price more than 4000000.", "answer": "SELECT T1.Writer FROM book AS T1 JOIN publication AS T2 ON T1.Book_ID = T2.Book_ID WHERE T2.Price > 4000000"}
|
||||
{"question": "Show the titles of books in descending order of publication price.", "answer": "SELECT T1.Title FROM book AS T1 JOIN publication AS T2 ON T1.Book_ID = T2.Book_ID ORDER BY T2.Price DESC NULLS LAST"}
|
||||
{"question": "Show publishers that have more than one publication.", "answer": "SELECT Publisher FROM publication GROUP BY Publisher HAVING COUNT(*) > 1"}
|
||||
{"question": "Show different publishers together with the number of publications they have.", "answer": "SELECT Publisher, COUNT(*) FROM publication GROUP BY Publisher"}
|
||||
{"question": "Please show the most common publication date.", "answer": "SELECT Publication_Date FROM publication GROUP BY Publication_Date ORDER BY COUNT(*) DESC NULLS LAST LIMIT 1"}
|
||||
{"question": "List the writers who have written more than one book.", "answer": "SELECT Writer FROM book GROUP BY Writer HAVING COUNT(*) > 1"}
|
||||
{"question": "List the titles of books that are not published.", "answer": "SELECT Title FROM book WHERE NOT Book_ID IN (SELECT Book_ID FROM publication)"}
|
||||
{"question": "Show the publishers that have publications with price higher than 10000000 and publications with price lower than 5000000.", "answer": "SELECT Publisher FROM publication WHERE Price > 10000000 INTERSECT SELECT Publisher FROM publication WHERE Price < 5000000"}
|
||||
{"question": "What is the number of distinct publication dates?", "answer": "SELECT COUNT(DISTINCT Publication_Date) FROM publication"}
|
||||
{"question": "How many distinct publication dates are there in our record?", "answer": "SELECT COUNT(DISTINCT Publication_Date) FROM publication"}
|
||||
{"question": "Show the prices of publications whose publisher is either \"Person\" or \"Wiley\"", "answer": "SELECT Price FROM publication WHERE Publisher = \"Person\" OR Publisher = \"Wiley\""}
|
||||
@@ -0,0 +1,100 @@
|
||||
{
|
||||
"catalog": "canner-cml",
|
||||
"schema": "spider",
|
||||
"models": [
|
||||
{
|
||||
"name": "book",
|
||||
"properties": {},
|
||||
"refSql": "select * from \"canner-cml\".spider.\"book_2-book\"",
|
||||
"columns": [
|
||||
{
|
||||
"name": "Book_ID",
|
||||
"type": "INTEGER",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Book_ID",
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"name": "Title",
|
||||
"type": "VARCHAR",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Title",
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"name": "Issues",
|
||||
"type": "REAL",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Issues",
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"name": "Writer",
|
||||
"type": "VARCHAR",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Writer",
|
||||
"properties": {}
|
||||
}
|
||||
],
|
||||
"primaryKey": ""
|
||||
},
|
||||
{
|
||||
"name": "publication",
|
||||
"properties": {},
|
||||
"refSql": "select * from \"canner-cml\".spider.\"book_2-publication\"",
|
||||
"columns": [
|
||||
{
|
||||
"name": "Publication_ID",
|
||||
"type": "INTEGER",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Publication_ID",
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"name": "Book_ID",
|
||||
"type": "INTEGER",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Book_ID",
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"name": "Publisher",
|
||||
"type": "VARCHAR",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Publisher",
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"name": "Publication_Date",
|
||||
"type": "VARCHAR",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Publication_Date",
|
||||
"properties": {}
|
||||
},
|
||||
{
|
||||
"name": "Price",
|
||||
"type": "REAL",
|
||||
"notNull": false,
|
||||
"isCalculated": false,
|
||||
"expression": "Price",
|
||||
"properties": {}
|
||||
}
|
||||
],
|
||||
"primaryKey": ""
|
||||
}
|
||||
],
|
||||
"relationships": [],
|
||||
"metrics": [],
|
||||
"cumulativeMetrics": [],
|
||||
"enumDefinitions": [],
|
||||
"views": [],
|
||||
"macros": []
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
import streamlit as st
|
||||
|
||||
|
||||
def show_eval_results(eval_file_path: Path):
|
||||
with open(eval_file_path, "r") as f:
|
||||
eval_results = json.load(f)
|
||||
|
||||
st.markdown("**Performance Benchmark**")
|
||||
for k, v in eval_results["eval_results"].items():
|
||||
if k == "details":
|
||||
continue
|
||||
st.markdown(f"**{k}**")
|
||||
if isinstance(v, dict):
|
||||
st.json(v, expanded=True)
|
||||
else:
|
||||
st.markdown(v)
|
||||
|
||||
|
||||
def get_question_right_or_wrong_mapping(eval_results: Dict) -> Dict:
|
||||
question_right_or_wrong_mapping = {}
|
||||
|
||||
for correct_type, correct_details in eval_results["details"]["correct"].items():
|
||||
for details in correct_details:
|
||||
question_right_or_wrong_mapping[details["question"]] = {
|
||||
"correct": True,
|
||||
"type": correct_type,
|
||||
"ground_truth_answer": details["ground_truth_answer"],
|
||||
"prediction_answer": details["prediction_answer"],
|
||||
"ragas_eval_results": details["ragas_eval_results"],
|
||||
}
|
||||
|
||||
for wrong_details in eval_results["details"]["wrong"]:
|
||||
question_right_or_wrong_mapping[wrong_details["question"]] = {
|
||||
"correct": False,
|
||||
"type": None,
|
||||
"ground_truth_answer": wrong_details["ground_truth_answer"],
|
||||
"prediction_answer": wrong_details["prediction_answer"],
|
||||
"ground_truth_query_results": wrong_details["ground_truth_query_results"],
|
||||
"prediction_query_results": wrong_details["prediction_query_results"],
|
||||
"prediction_error_details": wrong_details["prediction_error_details"],
|
||||
"ragas_eval_results": wrong_details["ragas_eval_results"],
|
||||
}
|
||||
|
||||
return question_right_or_wrong_mapping
|
||||
|
||||
|
||||
def show_single_prediction_result(
|
||||
question_right_or_wrong_mapping: Dict, prediction: Dict
|
||||
):
|
||||
expander_name = (
|
||||
f'(O) Question: {prediction["question"]}'
|
||||
if question_right_or_wrong_mapping[prediction["question"]]["correct"]
|
||||
else f'(X) Question: {prediction["question"]}'
|
||||
)
|
||||
|
||||
with st.expander(expander_name):
|
||||
if question_right_or_wrong_mapping[prediction["question"]]["correct"]:
|
||||
st.markdown("Correct type")
|
||||
st.text(question_right_or_wrong_mapping[prediction["question"]]["type"])
|
||||
st.markdown("Documents")
|
||||
st.json(prediction["contexts"], expanded=False)
|
||||
st.markdown("Answer")
|
||||
st.code(prediction["answer"], language="sql")
|
||||
st.markdown("Groud truth answer")
|
||||
st.code(
|
||||
question_right_or_wrong_mapping[prediction["question"]][
|
||||
"ground_truth_answer"
|
||||
],
|
||||
language="sql",
|
||||
)
|
||||
if not question_right_or_wrong_mapping[prediction["question"]]["correct"]:
|
||||
st.markdown("Ground truth query results")
|
||||
st.markdown(
|
||||
question_right_or_wrong_mapping[prediction["question"]][
|
||||
"ground_truth_query_results"
|
||||
]
|
||||
or "None"
|
||||
)
|
||||
st.markdown("Prediction query results")
|
||||
st.markdown(
|
||||
question_right_or_wrong_mapping[prediction["question"]][
|
||||
"prediction_query_results"
|
||||
]
|
||||
or "None"
|
||||
)
|
||||
st.markdown("Prediction error details")
|
||||
st.markdown(
|
||||
question_right_or_wrong_mapping[prediction["question"]][
|
||||
"prediction_error_details"
|
||||
]
|
||||
or "None"
|
||||
)
|
||||
if question_right_or_wrong_mapping[prediction["question"]][
|
||||
"ragas_eval_results"
|
||||
]:
|
||||
st.markdown("Ragas evaluation results")
|
||||
st.json(
|
||||
question_right_or_wrong_mapping[prediction["question"]][
|
||||
"ragas_eval_results"
|
||||
],
|
||||
expanded=False,
|
||||
)
|
||||
st.markdown("Metadata")
|
||||
st.json(prediction["metadata"], expanded=False)
|
||||
|
||||
|
||||
def show_prediction_results(eval_file_paths: List, prediction_file_paths: List):
|
||||
assert len(eval_file_paths) == len(prediction_file_paths) == 2
|
||||
|
||||
file_evals = []
|
||||
file_predictions = []
|
||||
|
||||
for eval_file_path in eval_file_paths:
|
||||
with open(eval_file_path, "r") as f:
|
||||
file_evals.append(json.load(f))
|
||||
|
||||
for prediction_file_path in prediction_file_paths:
|
||||
with open(prediction_file_path, "r") as f:
|
||||
file_predictions.append([json.loads(line) for line in f])
|
||||
|
||||
file1_question_right_or_wrong_mapping = get_question_right_or_wrong_mapping(
|
||||
file_evals[0]["eval_results"]
|
||||
)
|
||||
file2_question_right_or_wrong_mapping = get_question_right_or_wrong_mapping(
|
||||
file_evals[1]["eval_results"]
|
||||
)
|
||||
|
||||
predictions_correct_types_mapping = defaultdict(list)
|
||||
for i, (prediction1, prediction2) in enumerate(
|
||||
zip(file_predictions[0], file_predictions[1])
|
||||
):
|
||||
prediction1_correct = file1_question_right_or_wrong_mapping[
|
||||
prediction1["question"]
|
||||
]["correct"]
|
||||
prediction2_correct = file2_question_right_or_wrong_mapping[
|
||||
prediction2["question"]
|
||||
]["correct"]
|
||||
|
||||
if prediction1_correct and prediction2_correct:
|
||||
predictions_correct_types_mapping["both_correct"].append(i)
|
||||
elif not prediction1_correct and not prediction2_correct:
|
||||
predictions_correct_types_mapping["both_wrong"].append(i)
|
||||
elif prediction1_correct and not prediction2_correct:
|
||||
predictions_correct_types_mapping[
|
||||
"prediction1_correct_prediction2_wrong"
|
||||
].append(i)
|
||||
elif not prediction1_correct and prediction2_correct:
|
||||
predictions_correct_types_mapping[
|
||||
"prediction1_wrong_prediction2_correct"
|
||||
].append(i)
|
||||
|
||||
st.markdown("### Details")
|
||||
|
||||
st.markdown(
|
||||
f'#### Prediction 1 wrong, Prediction 2 correct ({len(predictions_correct_types_mapping['prediction1_wrong_prediction2_correct'])})'
|
||||
)
|
||||
col1, col2 = st.columns(2)
|
||||
for i in predictions_correct_types_mapping["prediction1_wrong_prediction2_correct"]:
|
||||
with col1:
|
||||
show_single_prediction_result(
|
||||
file1_question_right_or_wrong_mapping, file_predictions[0][i]
|
||||
)
|
||||
with col2:
|
||||
show_single_prediction_result(
|
||||
file2_question_right_or_wrong_mapping, file_predictions[1][i]
|
||||
)
|
||||
|
||||
st.markdown(
|
||||
f'#### Prediction 1 correct, Prediction 2 wrong ({len(predictions_correct_types_mapping['prediction1_correct_prediction2_wrong'])})'
|
||||
)
|
||||
col1, col2 = st.columns(2)
|
||||
for i in predictions_correct_types_mapping["prediction1_correct_prediction2_wrong"]:
|
||||
with col1:
|
||||
show_single_prediction_result(
|
||||
file1_question_right_or_wrong_mapping, file_predictions[0][i]
|
||||
)
|
||||
with col2:
|
||||
show_single_prediction_result(
|
||||
file2_question_right_or_wrong_mapping, file_predictions[1][i]
|
||||
)
|
||||
|
||||
st.markdown(
|
||||
f'#### Prediction 1 wrong, Prediction 2 wrong ({len(predictions_correct_types_mapping['both_wrong'])})'
|
||||
)
|
||||
col1, col2 = st.columns(2)
|
||||
for i in predictions_correct_types_mapping["both_wrong"]:
|
||||
with col1:
|
||||
show_single_prediction_result(
|
||||
file1_question_right_or_wrong_mapping, file_predictions[0][i]
|
||||
)
|
||||
with col2:
|
||||
show_single_prediction_result(
|
||||
file2_question_right_or_wrong_mapping, file_predictions[1][i]
|
||||
)
|
||||
|
||||
st.markdown(
|
||||
f'#### Prediction 1 correct, Prediction 2 correct ({len(predictions_correct_types_mapping['both_correct'])})'
|
||||
)
|
||||
col1, col2 = st.columns(2)
|
||||
for i in predictions_correct_types_mapping["both_correct"]:
|
||||
with col1:
|
||||
show_single_prediction_result(
|
||||
file1_question_right_or_wrong_mapping, file_predictions[0][i]
|
||||
)
|
||||
with col2:
|
||||
show_single_prediction_result(
|
||||
file2_question_right_or_wrong_mapping, file_predictions[1][i]
|
||||
)
|
||||
|
||||
|
||||
st.set_page_config(layout="wide")
|
||||
|
||||
st.title("Ask pipeline prediction evaluation results comparison")
|
||||
|
||||
output_dir = Path("outputs")
|
||||
output_files = list(output_dir.glob("*.json"))
|
||||
|
||||
# get dataset names
|
||||
dataset_names = sorted(
|
||||
set(
|
||||
[
|
||||
f.stem.split("_eval_results_")[0]
|
||||
if "_eval_results_" in f.stem
|
||||
else f.stem.split("_predictions_")[0]
|
||||
for f in output_files
|
||||
]
|
||||
)
|
||||
)
|
||||
dataset_name = st.selectbox("Select dataset", dataset_names)
|
||||
|
||||
# get timestamp in filename for selected dataset
|
||||
timestamps = sorted(
|
||||
set([f.stem.split("_")[-1] for f in output_files if dataset_name in f.stem])
|
||||
)
|
||||
selected_timestamps = st.multiselect(
|
||||
"Select 2 timestamps", timestamps, max_selections=2, disabled=len(timestamps) < 2
|
||||
)
|
||||
|
||||
st.markdown("---")
|
||||
|
||||
if len(selected_timestamps) == 2:
|
||||
prediction_file1_path = (
|
||||
output_dir / f"{dataset_name}_predictions_{selected_timestamps[0]}.json"
|
||||
)
|
||||
prediction_file2_path = (
|
||||
output_dir / f"{dataset_name}_predictions_{selected_timestamps[1]}.json"
|
||||
)
|
||||
eval_file1_path = (
|
||||
output_dir / f"{dataset_name}_eval_results_{selected_timestamps[0]}.json"
|
||||
)
|
||||
eval_file2_path = (
|
||||
output_dir / f"{dataset_name}_eval_results_{selected_timestamps[1]}.json"
|
||||
)
|
||||
|
||||
col1, col2 = st.columns(2)
|
||||
with col1:
|
||||
st.markdown(f"## {selected_timestamps[0]}")
|
||||
show_eval_results(eval_file1_path)
|
||||
with col2:
|
||||
st.markdown(f"## {selected_timestamps[1]}")
|
||||
show_eval_results(eval_file2_path)
|
||||
|
||||
show_prediction_results(
|
||||
eval_file_paths=[eval_file1_path, eval_file2_path],
|
||||
prediction_file_paths=[prediction_file1_path, prediction_file2_path],
|
||||
)
|
||||
@@ -39,7 +39,7 @@ def execute_sql_query(sql_query: str, db_path: str):
|
||||
|
||||
try:
|
||||
cur.execute(sql_query)
|
||||
# make each row a tuple of strings for easier comparison with the results from Vulcan
|
||||
# make each row a tuple of strings for easier comparison with the results from wren-engine
|
||||
# also sort each row to make the order of the columns consistent
|
||||
results = tuple(tuple(sorted(map(str, row))) for row in cur.fetchall())
|
||||
except Exception:
|
||||
@@ -51,7 +51,7 @@ def execute_sql_query(sql_query: str, db_path: str):
|
||||
return results
|
||||
|
||||
|
||||
def execute_sql_query_through_vulcan(sql_query: str):
|
||||
def execute_sql_query_through_wren_engine(sql_query: str):
|
||||
command = f'psql -d "postgres://localhost:7432/canner-cml?options=--search_path%3Dspider" -c "{sql_query}"'
|
||||
process = subprocess.Popen(
|
||||
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
|
||||
@@ -189,7 +189,7 @@ def generate_eval_report(
|
||||
"question": ground_truth["question"],
|
||||
"ground_truth_answer": ground_truth["answer"],
|
||||
"prediction_answer": prediction["answer"],
|
||||
"regas_eval_results": get_ragas_eval_results(
|
||||
"ragas_eval_results": get_ragas_eval_results(
|
||||
ragas_eval_results,
|
||||
i,
|
||||
),
|
||||
@@ -206,7 +206,7 @@ def generate_eval_report(
|
||||
(
|
||||
prediction_query_results,
|
||||
prediction_error_details,
|
||||
) = execute_sql_query_through_vulcan(
|
||||
) = execute_sql_query_through_wren_engine(
|
||||
prediction["answer"],
|
||||
)
|
||||
if len(ground_truth_query_results) == len(prediction_query_results):
|
||||
@@ -219,7 +219,7 @@ def generate_eval_report(
|
||||
"question": question,
|
||||
"ground_truth_answer": ground_truth["answer"],
|
||||
"prediction_answer": prediction["answer"],
|
||||
"regas_eval_results": get_ragas_eval_results(
|
||||
"ragas_eval_results": get_ragas_eval_results(
|
||||
ragas_eval_results,
|
||||
i,
|
||||
),
|
||||
@@ -239,7 +239,7 @@ def generate_eval_report(
|
||||
"prediction_answer": prediction["answer"],
|
||||
"ground_truth_query_results": ground_truth_query_results,
|
||||
"prediction_query_results": prediction_query_results,
|
||||
"regas_eval_results": get_ragas_eval_results(
|
||||
"ragas_eval_results": get_ragas_eval_results(
|
||||
ragas_eval_results,
|
||||
i,
|
||||
),
|
||||
@@ -255,7 +255,7 @@ def generate_eval_report(
|
||||
"ground_truth_query_results": ground_truth_query_results,
|
||||
"prediction_query_results": prediction_query_results,
|
||||
"prediction_error_details": prediction_error_details,
|
||||
"regas_eval_results": get_ragas_eval_results(
|
||||
"ragas_eval_results": get_ragas_eval_results(
|
||||
ragas_eval_results,
|
||||
i,
|
||||
),
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
PLATFORM=linux/arm64/v8
|
||||
CONFIG_PATH=./config.properties
|
||||
LAUNCH_CLI_PATH=./launch-cli.sh
|
||||
COMPOSE_PROJECT_NAME=wren
|
||||
Reference in New Issue
Block a user