diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts index a454f700c..ff8b88040 100644 --- a/apps/api/src/controllers/v0/crawl-cancel.ts +++ b/apps/api/src/controllers/v0/crawl-cancel.ts @@ -26,6 +26,9 @@ export async function crawlCancelController(req: Request, res: Response) { redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); + + redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + req.params.jobId + ":cancel") + .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id })); const sc = await getCrawl(req.params.jobId); if (!sc) { diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts index 9f6e5b7bb..78c2ae0b9 100644 --- a/apps/api/src/controllers/v0/crawl-status.ts +++ b/apps/api/src/controllers/v0/crawl-status.ts @@ -87,6 +87,9 @@ export async function crawlStatusController(req: Request, res: Response) { redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); + + redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + req.params.jobId + ":status") + .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id })); const sc = await getCrawl(req.params.jobId); if (!sc) { diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts index abd858a91..63b90e809 100644 --- a/apps/api/src/controllers/v0/crawl.ts +++ b/apps/api/src/controllers/v0/crawl.ts @@ -45,8 +45,13 @@ export async function crawlController(req: Request, res: Response) { return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); } + const id = uuidv4(); + redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); + + redisEvictConnection.sadd("teams_using_v0:" + team_id, "crawl:" + id) + .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id })); if (req.headers["x-idempotency-key"]) { const isIdempotencyValid = await validateIdempotencyKey(req); @@ -154,8 +159,6 @@ export async function crawlController(req: Request, res: Response) { // } // } - const id = uuidv4(); - await logCrawl(id, team_id); const { scrapeOptions, internalOptions } = fromLegacyScrapeOptions( diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.ts index febac4115..87179e19f 100644 --- a/apps/api/src/controllers/v0/keyAuth.ts +++ b/apps/api/src/controllers/v0/keyAuth.ts @@ -19,6 +19,9 @@ export const keyAuthController = async (req: Request, res: Response) => { redisEvictConnection.sadd("teams_using_v0", auth.team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id: auth.team_id })); + + redisEvictConnection.sadd("teams_using_v0:" + auth.team_id, "keyAuth") + .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id: auth.team_id })); // if success, return success: true return res.status(200).json({ success: true }); diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts index d4bd73bbc..3e02e4e3d 100644 --- a/apps/api/src/controllers/v0/scrape.ts +++ b/apps/api/src/controllers/v0/scrape.ts @@ -181,9 +181,14 @@ export async function scrapeController(req: Request, res: Response) { return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); } + const jobId = uuidv4(); + redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); + redisEvictConnection.sadd("teams_using_v0:" + team_id, "scrape:" + jobId) + .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id })); + const crawlerOptions = req.body.crawlerOptions ?? {}; const pageOptions = { ...defaultPageOptions, ...req.body.pageOptions }; const extractorOptions = { @@ -228,8 +233,6 @@ export async function scrapeController(req: Request, res: Response) { }); } - const jobId = uuidv4(); - const startTime = new Date().getTime(); const result = await scrapeHelper( jobId, diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts index aaff178c2..2341bac45 100644 --- a/apps/api/src/controllers/v0/search.ts +++ b/apps/api/src/controllers/v0/search.ts @@ -174,9 +174,14 @@ export async function searchController(req: Request, res: Response) { return res.status(400).json({ error: "Your team has zero data retention enabled. This is not supported on the v0 API. Please update your code to use the v1 API." }); } + const jobId = uuidv4(); + redisEvictConnection.sadd("teams_using_v0", team_id) .catch(error => logger.error("Failed to add team to teams_using_v0", { error, team_id })); + redisEvictConnection.sadd("teams_using_v0:" + team_id, "search:" + jobId) + .catch(error => logger.error("Failed to add team to teams_using_v0 (2)", { error, team_id })); + const crawlerOptions = req.body.crawlerOptions ?? {}; const pageOptions = req.body.pageOptions ?? { includeHtml: req.body.pageOptions?.includeHtml ?? false, @@ -189,8 +194,6 @@ export async function searchController(req: Request, res: Response) { const searchOptions = req.body.searchOptions ?? { limit: 5 }; - const jobId = uuidv4(); - try { const { success: creditsCheckSuccess, message: creditsCheckMessage } = await checkTeamCredits(chunk, team_id, 1);