only write if the metadata changes

.changeset/sweet-ligers-sing.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Include model and apiProvider in metadata for cline to read

src/core/context-tracking/ContextTrackerTypes.ts

@@ -10,8 +10,16 @@ export interface FileMetadataEntry {
 	user_edit_date?: number | null
 }
 
+export interface ModelMetadataEntry {
+	ts: number
+	model_id: string
+	model_provider_id: string
+	mode: string
+}
+
 export interface TaskMetadata {
 	files_in_context: FileMetadataEntry[]
+	model_usage: ModelMetadataEntry[]
 }
 
 // Interface for the controller to avoid direct dependency

src/core/context-tracking/FileContextTracker.test.ts

@@ -5,7 +5,7 @@ import * as vscode from "vscode"
 import * as path from "path"
 import { FileContextTracker } from "./FileContextTracker"
 import * as diskModule from "../storage/disk"
-import type { TaskMetadata, ControllerLike, FileMetadataEntry } from "./FileContextTrackerTypes"
+import type { TaskMetadata, ControllerLike, FileMetadataEntry } from "./ContextTrackerTypes"
 
 describe("FileContextTracker", () => {
 	let sandbox: sinon.SinonSandbox
@@ -53,7 +53,7 @@ describe("FileContextTracker", () => {
 		}
 
 		// Mock disk module functions
-		mockTaskMetadata = { files_in_context: [] }
+		mockTaskMetadata = { files_in_context: [], model_usage: [] }
 		getTaskMetadataStub = sandbox.stub(diskModule, "getTaskMetadata").resolves(mockTaskMetadata)
 		saveTaskMetadataStub = sandbox.stub(diskModule, "saveTaskMetadata").resolves()
 

src/core/context-tracking/FileContextTracker.ts

@@ -1,7 +1,7 @@
 import * as path from "path"
 import * as vscode from "vscode"
 import { getTaskMetadata, saveTaskMetadata } from "../storage/disk"
-import type { FileMetadataEntry, ControllerLike } from "./FileContextTrackerTypes"
+import type { FileMetadataEntry, ControllerLike } from "./ContextTrackerTypes"
 
 // This class is responsible for tracking file operations that may result in stale context.
 // If a user modifies a file outside of Cline, the context may become stale and need to be updated.

src/core/context-tracking/ModelContextTracker.test.ts

@@ -0,0 +1,211 @@
+import { describe, it, beforeEach, afterEach } from "mocha"
+import { expect } from "chai"
+import * as sinon from "sinon"
+import * as vscode from "vscode"
+import { ModelContextTracker } from "./ModelContextTracker"
+import * as diskModule from "../storage/disk"
+import type { TaskMetadata, ControllerLike } from "./ContextTrackerTypes"
+
+describe("ModelContextTracker", () => {
+	let sandbox: sinon.SinonSandbox
+	let mockController: ControllerLike
+	let mockContext: vscode.ExtensionContext
+	let tracker: ModelContextTracker
+	let taskId: string
+	let mockTaskMetadata: TaskMetadata
+	let getTaskMetadataStub: sinon.SinonStub
+	let saveTaskMetadataStub: sinon.SinonStub
+
+	beforeEach(() => {
+		sandbox = sinon.createSandbox()
+
+		// Mock controller and context
+		mockContext = {
+			globalStorageUri: { fsPath: "/mock/storage" },
+		} as unknown as vscode.ExtensionContext
+
+		mockController = {
+			context: mockContext,
+		}
+
+		// Mock disk module functions
+		mockTaskMetadata = { files_in_context: [], model_usage: [] }
+		getTaskMetadataStub = sandbox.stub(diskModule, "getTaskMetadata").resolves(mockTaskMetadata)
+		saveTaskMetadataStub = sandbox.stub(diskModule, "saveTaskMetadata").resolves()
+
+		// Create tracker instance
+		taskId = "test-task-id"
+		tracker = new ModelContextTracker(mockController, taskId)
+	})
+
+	afterEach(() => {
+		sandbox.restore()
+	})
+
+	it("should record model usage with correct data", async () => {
+		// Test data
+		const apiProviderId = "anthropic"
+		const modelId = "claude-3-opus"
+		const mode = "act"
+
+		// Use a fake timer to have a predictable timestamp
+		const fakeNow = 1617293940000 // Some fixed timestamp
+		const clock = sandbox.useFakeTimers(fakeNow)
+
+		try {
+			// Call the method being tested
+			await tracker.recordModelUsage(apiProviderId, modelId, mode)
+
+			// Verify getTaskMetadata was called with correct parameters
+			expect(getTaskMetadataStub.calledOnce).to.be.true
+			expect(getTaskMetadataStub.firstCall.args[1]).to.equal(taskId)
+
+			// Verify saveTaskMetadata was called with the correct data
+			expect(saveTaskMetadataStub.calledOnce).to.be.true
+
+			// Extract the saved metadata from the call arguments
+			const savedMetadata = saveTaskMetadataStub.firstCall.args[2]
+
+			// Verify model_usage array has one entry
+			expect(savedMetadata.model_usage.length).to.equal(1)
+
+			// Verify the entry has the correct properties
+			const modelUsageEntry = savedMetadata.model_usage[0]
+			expect(modelUsageEntry.ts).to.equal(fakeNow)
+			expect(modelUsageEntry.model_id).to.equal(modelId)
+			expect(modelUsageEntry.model_provider_id).to.equal(apiProviderId)
+			expect(modelUsageEntry.mode).to.equal(mode)
+		} finally {
+			// Restore the clock
+			clock.restore()
+		}
+	})
+
+	it("should throw an error when controller is dereferenced", async () => {
+		// Create a new tracker with a controller that will be garbage collected
+		const weakMockController = { context: mockContext }
+		const weakTracker = new ModelContextTracker(weakMockController, taskId)
+
+		// Force the WeakRef to return null by overriding the deref method
+		const weakRef = { deref: sandbox.stub().returns(null) }
+		sandbox.stub(WeakRef.prototype, "deref").callsFake(() => weakRef.deref())
+
+		try {
+			// Try to call the method - this should throw
+			await weakTracker.recordModelUsage("any-provider", "any-model", "any-mode")
+
+			// If we get here, the test should fail
+			expect.fail("Expected an error to be thrown")
+		} catch (error) {
+			// Verify the error message
+			expect(error.message).to.equal("Unable to access extension context")
+		}
+	})
+
+	it("should append model usage to existing entries", async () => {
+		// Add an existing model usage entry
+		const existingTimestamp = 1617200000000
+		mockTaskMetadata.model_usage = [
+			{
+				ts: existingTimestamp,
+				model_id: "existing-model",
+				model_provider_id: "existing-provider",
+				mode: "plan",
+			},
+		]
+
+		// Test data for new entry
+		const apiProviderId = "anthropic"
+		const modelId = "claude-3-sonnet"
+		const mode = "act"
+
+		// Use a fake timer
+		const newTimestamp = 1617300000000
+		const clock = sandbox.useFakeTimers(newTimestamp)
+
+		try {
+			// Call the method being tested
+			await tracker.recordModelUsage(apiProviderId, modelId, mode)
+
+			// Verify saveTaskMetadata was called
+			expect(saveTaskMetadataStub.calledOnce).to.be.true
+
+			// Extract the saved metadata
+			const savedMetadata = saveTaskMetadataStub.firstCall.args[2]
+
+			// Verify model_usage array now has two entries
+			expect(savedMetadata.model_usage.length).to.equal(2)
+
+			// Verify the existing entry is preserved
+			expect(savedMetadata.model_usage[0]).to.deep.equal({
+				ts: existingTimestamp,
+				model_id: "existing-model",
+				model_provider_id: "existing-provider",
+				mode: "plan",
+			})
+
+			// Verify the new entry has correct data
+			expect(savedMetadata.model_usage[1]).to.deep.equal({
+				ts: newTimestamp,
+				model_id: modelId,
+				model_provider_id: apiProviderId,
+				mode: mode,
+			})
+		} finally {
+			clock.restore()
+		}
+	})
+
+	it("should handle multiple model usages in sequence", async () => {
+		// Test data for sequential calls
+		const usages = [
+			{ provider: "anthropic", model: "claude-3-opus", mode: "plan" },
+			{ provider: "openai", model: "gpt-4", mode: "act" },
+			{ provider: "anthropic", model: "claude-3-haiku", mode: "plan" },
+		]
+
+		// Use a fake timer that advances with each call
+		const startTime = 1617300000000
+		const clock = sandbox.useFakeTimers(startTime)
+
+		try {
+			// Record multiple model usages
+			for (let i = 0; i < usages.length; i++) {
+				const { provider, model, mode } = usages[i]
+
+				// Advance time by 1 second for each call
+				clock.tick(1000)
+				const expectedTime = startTime + (i + 1) * 1000
+
+				// Reset history between calls to check individual call behavior
+				getTaskMetadataStub.resetHistory()
+				saveTaskMetadataStub.resetHistory()
+
+				// Reset mock metadata for each iteration to avoid accumulation
+				mockTaskMetadata.model_usage = []
+
+				// Call the method
+				await tracker.recordModelUsage(provider, model, mode)
+
+				// Verify interaction with disk module
+				expect(getTaskMetadataStub.calledOnce).to.be.true
+				expect(saveTaskMetadataStub.calledOnce).to.be.true
+
+				// Get the saved metadata
+				const savedMetadata = saveTaskMetadataStub.firstCall.args[2]
+
+				// Since we reset the array for each call, we should always have 1 entry
+				expect(savedMetadata.model_usage.length).to.equal(1)
+
+				// Check the entry
+				const entry = savedMetadata.model_usage[0]
+				expect(entry.ts).to.equal(expectedTime)
+				expect(entry.model_id).to.equal(model)
+				expect(entry.model_provider_id).to.equal(provider)
+				expect(entry.mode).to.equal(mode)
+			}
+		} finally {
+			clock.restore()
+		}
+	})
+})

src/core/context-tracking/ModelContextTracker.ts

@@ -0,0 +1,52 @@
+import * as vscode from "vscode"
+import { getTaskMetadata, saveTaskMetadata } from "../storage/disk"
+import type { ControllerLike } from "./ContextTrackerTypes"
+
+export class ModelContextTracker {
+	readonly taskId: string
+	private controllerRef: WeakRef<ControllerLike>
+
+	constructor(controller: ControllerLike, taskId: string) {
+		this.controllerRef = new WeakRef(controller)
+		this.taskId = taskId
+	}
+
+	// While a task is ref'd by a controller, it will always have access to the extension context
+	// This error is thrown if the controller derefs the task after e.g., aborting the task
+	private context(): vscode.ExtensionContext {
+		const context = this.controllerRef.deref()?.context
+		if (!context) {
+			throw new Error("Unable to access extension context")
+		}
+		return context
+	}
+
+	async recordModelUsage(apiProviderId: string, modelId: string, mode: string) {
+		const context = this.context()
+		const metadata = await getTaskMetadata(context, this.taskId)
+
+		if (!metadata.model_usage) {
+			metadata.model_usage = []
+		}
+
+		// check to see if the last entry is the same as the new one
+		const lastEntry = metadata.model_usage[metadata.model_usage.length - 1]
+		if (
+			lastEntry &&
+			lastEntry.model_id === modelId &&
+			lastEntry.model_provider_id === apiProviderId &&
+			lastEntry.mode === mode
+		) {
+			return
+		}
+
+		metadata.model_usage.push({
+			ts: Date.now(),
+			model_id: modelId,
+			model_provider_id: apiProviderId,
+			mode: mode,
+		})
+
+		await saveTaskMetadata(context, this.taskId, metadata)
+	}
+}

src/core/storage/disk.ts

@@ -4,19 +4,7 @@ import fs from "fs/promises"
 import { Anthropic } from "@anthropic-ai/sdk"
 import { fileExistsAtPath } from "../../utils/fs"
 import { ClineMessage } from "../../shared/ExtensionMessage"
-
-export interface FileMetadataEntry {
-	path: string
-	record_state: "active" | "stale"
-	record_source: "read_tool" | "user_edited" | "cline_edited" | "file_mentioned"
-	cline_read_date: number | null
-	cline_edit_date: number | null
-	user_edit_date?: number | null
-}
-
-export interface TaskMetadata {
-	files_in_context: FileMetadataEntry[]
-}
+import type { TaskMetadata } from "../context-tracking/ContextTrackerTypes"
 
 export const GlobalFileNames = {
 	apiConversationHistory: "api_conversation_history.json",
@@ -96,7 +84,7 @@ export async function getTaskMetadata(context: vscode.ExtensionContext, taskId:
 	} catch (error) {
 		console.error("Failed to read task metadata:", error)
 	}
-	return { files_in_context: [] }
+	return { files_in_context: [], model_usage: [] }
 }
 
 export async function saveTaskMetadata(context: vscode.ExtensionContext, taskId: string, metadata: TaskMetadata) {

src/core/task/index.ts

@@ -66,6 +66,8 @@ import { parseMentions } from ".././mentions"
 import { formatResponse } from ".././prompts/responses"
 import { addUserInstructions, SYSTEM_PROMPT } from ".././prompts/system"
 import { FileContextTracker } from "../context-tracking/FileContextTracker"
+import { ModelContextTracker } from "../context-tracking/ModelContextTracker"
+
 import {
 	checkIsAnthropicContextWindowError,
 	checkIsOpenRouterContextWindowError,
@@ -78,6 +80,7 @@ import {
 	saveApiConversationHistory,
 	saveClineMessages,
 	GlobalFileNames,
+	getTaskMetadata,
 } from "../storage/disk"
 
 const cwd = vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath).at(0) ?? path.join(os.homedir(), "Desktop") // may or may not exist but fs checking existence would immediately ask for permission which would be bad UX, need to come up with a better solution
@@ -119,8 +122,9 @@ export class Task {
 	isAwaitingPlanResponse = false
 	didRespondToPlanAskBySwitchingMode = false
 
-	// File tracking
+	// Metadata tracking
 	private fileContextTracker: FileContextTracker
+	private modelContextTracker: ModelContextTracker
 
 	// streaming
 	isWaitingForFirstChunk = false
@@ -175,7 +179,7 @@ export class Task {
 
 		// Initialize file context tracker
 		this.fileContextTracker = new FileContextTracker(controller, this.taskId)
-
+		this.modelContextTracker = new ModelContextTracker(controller, this.taskId)
 		// Now that taskId is initialized, we can build the API handler
 		this.api = buildApiHandler({
 			...apiConfiguration,
@@ -3072,6 +3076,10 @@ export class Task {
 			throw new Error("Cline instance aborted")
 		}
 
+		if (this.apiProvider && this.api.getModel().id) {
+			await this.modelContextTracker.recordModelUsage(this.apiProvider, this.api.getModel().id, this.chatSettings.mode)
+		}
+
 		if (this.consecutiveMistakeCount >= 3) {
 			if (this.autoApprovalSettings.enabled && this.autoApprovalSettings.enableNotifications) {
 				showSystemNotification({