mirror of
https://github.com/cline/cline.git
synced 2026-09-01 15:11:04 +08:00
Compare commits
100 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8f1c70f3fb | |||
| 1676b37ff0 | |||
| 3cbc224176 | |||
| 25e98e6c40 | |||
| 53faa4a964 | |||
| 9da843573c | |||
| e3b425aa97 | |||
| c96d16b4c8 | |||
| 7efacc900e | |||
| a99e80d1ee | |||
| 419b22f501 | |||
| 983df312c0 | |||
| 9b91caf658 | |||
| 9924c94b5b | |||
| ff9e87182c | |||
| d31a05f13a | |||
| 9bbcb4ed07 | |||
| dbab8d62f3 | |||
| 54fe51bf9d | |||
| 6e7df5c28d | |||
| bd4801a4af | |||
| 9d662325ce | |||
| c1437a933d | |||
| f1821790ba | |||
| cf8248aaf7 | |||
| 87a72e2a95 | |||
| e88c92ad3f | |||
| 2ebc2421d8 | |||
| 5550803609 | |||
| 6ecd8c265d | |||
| 05e3e2e915 | |||
| 2b7147dcc0 | |||
| 309f384694 | |||
| 9cbe5d08f1 | |||
| f7544f7def | |||
| bfd3d7a478 | |||
| 322f71b1f2 | |||
| e27eec5d7d | |||
| be1c99d2a7 | |||
| 61ff4fa182 | |||
| 261b6dffc1 | |||
| 1cd58966d9 | |||
| 0d07e1ace9 | |||
| 14246dd658 | |||
| 0514ebeaa9 | |||
| 97efddb6b5 | |||
| 922d5e06c7 | |||
| 6400f69b7e | |||
| 97a3ff8fca | |||
| 07fb212879 | |||
| ab1a6cc904 | |||
| 67ced789f8 | |||
| f892b19940 | |||
| 47084f4a7e | |||
| fd3a69d208 | |||
| be036dee41 | |||
| 63ccec790c | |||
| 83e8770fea | |||
| 749c758e69 | |||
| 214507ffb9 | |||
| eab1231d44 | |||
| 3b460b5072 | |||
| 929d6817fd | |||
| 7d507c8251 | |||
| 2950939908 | |||
| 83b365f7ea | |||
| 9158ecfa51 | |||
| 2fcba91fd0 | |||
| e5546eb73d | |||
| 30f946af65 | |||
| b683e8310c | |||
| 23722596a3 | |||
| dce792ca12 | |||
| 0c73fd8dfd | |||
| 720764463c | |||
| 093b245cb9 | |||
| faee588e27 | |||
| e53e2ab77a | |||
| aa59809887 | |||
| 868965107d | |||
| 0d16cf17d2 | |||
| a627919f15 | |||
| f9927697cf | |||
| 46c49436f8 | |||
| 9480607917 | |||
| fa9a529ed4 | |||
| 5c73eb2b33 | |||
| 6bfda15006 | |||
| 7e5be1c050 | |||
| 57cdb60e63 | |||
| 1afa0a1dbc | |||
| 2c77b18ee2 | |||
| 1a800d33b8 | |||
| d1e41b2a04 | |||
| e0ce2fa715 | |||
| df8196d200 | |||
| f034dd41b2 | |||
| a2f7671066 | |||
| d569d0a423 | |||
| a7e1643d6a |
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"claude-dev": patch
|
||||
---
|
||||
|
||||
Add speech-to-text dictation feature for Cline account users
|
||||
@@ -90,6 +90,7 @@
|
||||
"features/focus-chain",
|
||||
"features/auto-compact",
|
||||
"features/editing-messages",
|
||||
"features/dictation",
|
||||
{
|
||||
"group": "@ Mentions",
|
||||
"pages": [
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
---
|
||||
title: Dictation
|
||||
description:
|
||||
---
|
||||
|
||||
Cline lets you transcribe speech to text in an easy, built-in service
|
||||
|
||||
## Get Started
|
||||
|
||||
1. **Enable Dictation** in Feature Settings.
|
||||
2. **Click the microphone** in the chat input area.
|
||||
3. **Speak** - the button turns red while recording.
|
||||
4. **Click Stop Recording** when done.
|
||||
5. **Wait for transcription** - "[Transcribing...]" will appear and then the finished transcription will appear.
|
||||
|
||||
## Settings
|
||||
|
||||
Enable or disable dictation in Feature Settings by toggling "Enable Dictation." You can also change the transcription language to one of the supported languages.
|
||||
|
||||
## Requirements
|
||||
|
||||
Cline uses FFmpeg to capture your voice across all platforms:
|
||||
|
||||
- **macOS**: FFmpeg (via Homebrew: `brew install ffmpeg`)
|
||||
- **Linux**: FFmpeg (via apt: `sudo apt-get install ffmpeg`)
|
||||
- **Windows**: FFmpeg (via winget: `winget install Gyan.FFmpeg`)
|
||||
|
||||
If you don't have FFmpeg installed, Cline will automatically detect this and prompt you to install it with a single click.
|
||||
|
||||
## Technical Details
|
||||
|
||||
### Independent from Chat Provider
|
||||
|
||||
The voice transcription feature works completely independently from whatever chat provider you're using. You can use Claude, GPT-4, or any other model for your conversations, and voice transcription will always use Cline's own transcription service. As long as you have a valid Cline account with enough credits, dictation will work regardless of your chat model choice.
|
||||
|
||||
### Audio Format
|
||||
|
||||
Voice recordings are captured in WebM format using the Opus codec for optimal compression. The system records in mono (single channel) at a 16kHz sample rate, which is specifically optimized for voice recognition. The bitrate is set to 32kbps to keep file sizes efficient while maintaining good audio quality.
|
||||
|
||||
### Privacy & Security
|
||||
|
||||
Your audio is recorded locally on your machine and only the audio file itself is sent to Cline's transcription service for processing. No audio is stored anywhere after transcription is complete, and all temporary files are automatically cleaned up to protect your privacy.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
`Failed to start recording` - Audio recording tools aren't installed. Cline will prompt you to install FFmpeg - just follow the chat instructions.
|
||||
|
||||
`Invalid audio format or request data` - Usually an audio recording issue. Make sure FFmpeg is properly installed and working.
|
||||
|
||||
`Authentication failed` - You need to reauthenticate your Cline account. Sign out and back in, then check your internet connection.
|
||||
|
||||
`Insufficient credits for transcription service` - Your Cline account doesn't have enough credits. Check your balance and purchase more if needed.
|
||||
|
||||
`Cannot connect to transcription service` - Connection issue. Check your internet connection and firewall settings aren't blocking Cline's servers.
|
||||
|
||||
## API Usage
|
||||
|
||||
Voice transcription uses Cline's transcription service, which requires credits from your Cline account. Currently, voice transcription is billed at $0.006 per minute of audio.
|
||||
|
||||
**Note:** We are still experimenting with this feature and pricing may change in the future.
|
||||
@@ -0,0 +1,41 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package cline;
|
||||
import "cline/common.proto";
|
||||
option java_package = "bot.cline.proto";
|
||||
option java_multiple_files = true;
|
||||
|
||||
service DictationService {
|
||||
rpc startRecording(EmptyRequest) returns (RecordingResult);
|
||||
rpc stopRecording(EmptyRequest) returns (RecordedAudio);
|
||||
rpc cancelRecording(EmptyRequest) returns (RecordingResult);
|
||||
rpc getRecordingStatus(EmptyRequest) returns (RecordingStatus);
|
||||
rpc transcribeAudio(TranscribeAudioRequest) returns (Transcription);
|
||||
}
|
||||
|
||||
message TranscribeAudioRequest {
|
||||
string audio_base64 = 2;
|
||||
string language = 3;
|
||||
}
|
||||
|
||||
message RecordingResult {
|
||||
bool success = 1;
|
||||
string error = 2;
|
||||
}
|
||||
|
||||
message RecordedAudio {
|
||||
bool success = 1;
|
||||
string audio_base64 = 2;
|
||||
string error = 3;
|
||||
}
|
||||
|
||||
message RecordingStatus {
|
||||
bool is_recording = 1;
|
||||
double duration_seconds = 2;
|
||||
string error = 3;
|
||||
}
|
||||
|
||||
message Transcription {
|
||||
string text = 1;
|
||||
string error = 2;
|
||||
}
|
||||
@@ -123,9 +123,10 @@ message UpdateSettingsRequest {
|
||||
optional string preferred_language = 14;
|
||||
optional OpenaiReasoningEffort openai_reasoning_effort = 15;
|
||||
optional bool strict_plan_mode_enabled = 16;
|
||||
optional FocusChainSettings focus_chain_settings = 17;
|
||||
optional bool use_auto_condense = 18;
|
||||
optional string custom_prompt = 19;
|
||||
optional DictationSettings dictation_settings = 17;
|
||||
optional FocusChainSettings focus_chain_settings = 18;
|
||||
optional bool use_auto_condense = 19;
|
||||
optional string custom_prompt = 20;
|
||||
}
|
||||
|
||||
// Complete API Configuration message
|
||||
@@ -263,6 +264,11 @@ message UpdateTerminalConnectionTimeoutRequest {
|
||||
optional int32 timeout_ms = 1;
|
||||
}
|
||||
|
||||
message DictationSettings {
|
||||
bool dictation_enabled = 1;
|
||||
string dictation_language = 2;
|
||||
}
|
||||
|
||||
message FocusChainSettings {
|
||||
bool enabled = 1;
|
||||
int32 remind_cline_interval = 2;
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import { telemetryService } from "@services/posthog/PostHogClientProvider"
|
||||
import { RecordingResult } from "@shared/proto/cline/dictation"
|
||||
import { audioRecordingService } from "@/services/dictation/AudioRecordingService"
|
||||
import { Controller } from ".."
|
||||
|
||||
/**
|
||||
* Cancels audio recording without saving or transcribing the audio
|
||||
* @param controller The controller instance
|
||||
* @returns RecordingResult indicating success or failure
|
||||
*/
|
||||
export const cancelRecording = async (controller: Controller): Promise<RecordingResult> => {
|
||||
const taskId = controller.task?.taskId
|
||||
const recordingStatus = audioRecordingService.getRecordingStatus()
|
||||
const recordingDuration = recordingStatus.durationSeconds * 1000 // Convert to milliseconds
|
||||
|
||||
try {
|
||||
const result = await audioRecordingService.cancelRecording()
|
||||
|
||||
telemetryService.captureVoiceRecordingStopped(taskId, recordingDuration, false, process.platform)
|
||||
|
||||
return RecordingResult.create({
|
||||
success: result.success,
|
||||
error: result.error || "",
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Error canceling recording:", error)
|
||||
|
||||
telemetryService.captureVoiceRecordingStopped(taskId, recordingDuration, false, process.platform)
|
||||
|
||||
return RecordingResult.create({
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "Unknown error occurred",
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import { RecordingStatus } from "@shared/proto/cline/dictation"
|
||||
import { audioRecordingService } from "@/services/dictation/AudioRecordingService"
|
||||
|
||||
/**
|
||||
* Gets the current recording status
|
||||
* @returns RecordingStatus with current status
|
||||
*/
|
||||
export const getRecordingStatus = async (): Promise<RecordingStatus> => {
|
||||
try {
|
||||
const status = audioRecordingService.getRecordingStatus()
|
||||
|
||||
return RecordingStatus.create({
|
||||
isRecording: status.isRecording,
|
||||
durationSeconds: status.durationSeconds,
|
||||
error: status.error || "",
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Error getting recording status:", error)
|
||||
return RecordingStatus.create({
|
||||
isRecording: false,
|
||||
durationSeconds: 0,
|
||||
error: error instanceof Error ? error.message : "Unknown error occurred",
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import { RecordingResult } from "@shared/proto/cline/dictation"
|
||||
import * as os from "os"
|
||||
import { HostProvider } from "@/hosts/host-provider"
|
||||
import { audioRecordingService } from "@/services/dictation/AudioRecordingService"
|
||||
import { telemetryService } from "@/services/posthog/PostHogClientProvider"
|
||||
import { AUDIO_PROGRAM_CONFIG } from "@/shared/audioProgramConstants"
|
||||
import { ShowMessageType } from "@/shared/proto/host/window"
|
||||
import { Controller } from ".."
|
||||
|
||||
/**
|
||||
* Handles the installation of missing dependencies with Cline
|
||||
*/
|
||||
async function handleInstallWithCline(
|
||||
controller: Controller,
|
||||
dependencyName: string,
|
||||
installCommand: string,
|
||||
platform: string,
|
||||
): Promise<void> {
|
||||
const platformName = platform === "darwin" ? "macOS" : platform === "win32" ? "Windows" : "Linux"
|
||||
const installTask = `Please install ${dependencyName} for voice recording on ${platformName}.\n\nRun this command:\n\`\`\`bash\n${installCommand}\n\`\`\`\n\nThis will enable voice recording functionality in Cline.`
|
||||
|
||||
// Clear any existing task and start the installation task
|
||||
await controller.clearTask()
|
||||
await controller.postStateToWebview()
|
||||
await controller.initTask(installTask)
|
||||
|
||||
HostProvider.get().logToChannel(`Started task to install ${dependencyName}`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles copying the installation command to clipboard
|
||||
*/
|
||||
async function handleCopyCommand(installCommand: string): Promise<void> {
|
||||
const vscode = await import("vscode")
|
||||
await vscode.env.clipboard.writeText(installCommand)
|
||||
|
||||
await HostProvider.window.showMessage({
|
||||
type: ShowMessageType.INFORMATION,
|
||||
message: `Installation command copied to clipboard: ${installCommand}`,
|
||||
options: { items: [] },
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles missing dependency notification and user action
|
||||
*/
|
||||
async function handleMissingDependency(
|
||||
controller: Controller,
|
||||
platform: string,
|
||||
config: (typeof AUDIO_PROGRAM_CONFIG)[keyof typeof AUDIO_PROGRAM_CONFIG],
|
||||
): Promise<void> {
|
||||
const installWithCline = "Install with Cline"
|
||||
const installManually = "Copy Command"
|
||||
const dismiss = "Dismiss"
|
||||
|
||||
const action = await HostProvider.window.showMessage({
|
||||
type: ShowMessageType.INFORMATION,
|
||||
message: `${config.dependencyName} is required for voice recording. ${config.installDescription}`,
|
||||
options: { items: [installWithCline, installManually, dismiss] },
|
||||
})
|
||||
|
||||
if (action.selectedOption === installWithCline) {
|
||||
await handleInstallWithCline(controller, config.dependencyName, config.installCommand, platform)
|
||||
} else if (action.selectedOption === installManually) {
|
||||
await handleCopyCommand(config.installCommand)
|
||||
}
|
||||
// If dismiss, do nothing
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles sign-in errors for dictation
|
||||
*/
|
||||
async function handleSignInError(controller: Controller, errorMessage: string): Promise<void> {
|
||||
const signInAction = "Sign in to Cline"
|
||||
const action = await HostProvider.window.showMessage({
|
||||
type: ShowMessageType.ERROR,
|
||||
message: `Voice recording error: ${errorMessage}`,
|
||||
options: { items: [signInAction] },
|
||||
})
|
||||
|
||||
if (action.selectedOption === signInAction) {
|
||||
await controller.authService.createAuthRequest()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shows a generic error message
|
||||
*/
|
||||
async function showGenericError(errorMessage: string): Promise<void> {
|
||||
await HostProvider.window.showMessage({
|
||||
type: ShowMessageType.ERROR,
|
||||
message: `Voice recording error: ${errorMessage}`,
|
||||
options: { items: [] },
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the recording error is due to missing dependencies
|
||||
*/
|
||||
function isMissingDependencyError(
|
||||
error: string | undefined,
|
||||
config: (typeof AUDIO_PROGRAM_CONFIG)[keyof typeof AUDIO_PROGRAM_CONFIG] | undefined,
|
||||
): boolean {
|
||||
return !!(error && config && error.includes(config.error))
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts audio recording using the Extension Host
|
||||
* @param controller The controller instance
|
||||
* @returns RecordingResult with success status
|
||||
*/
|
||||
export const startRecording = async (controller: Controller): Promise<RecordingResult> => {
|
||||
const taskId = controller.task?.taskId
|
||||
|
||||
try {
|
||||
// Verify user authentication
|
||||
const userInfo = controller.authService.getInfo()
|
||||
if (!userInfo?.user?.uid) {
|
||||
throw new Error("Please sign in to your Cline Account to use Dictation.")
|
||||
}
|
||||
|
||||
// Attempt to start recording
|
||||
const result = await audioRecordingService.startRecording()
|
||||
|
||||
// Handle successful recording start
|
||||
if (result.success) {
|
||||
telemetryService.captureVoiceRecordingStarted(taskId, process.platform)
|
||||
return RecordingResult.create({
|
||||
success: true,
|
||||
error: "",
|
||||
})
|
||||
}
|
||||
|
||||
// Check if the error is due to missing dependencies
|
||||
const platform = os.platform() as keyof typeof AUDIO_PROGRAM_CONFIG
|
||||
const config = AUDIO_PROGRAM_CONFIG[platform]
|
||||
|
||||
if (isMissingDependencyError(result.error, config)) {
|
||||
await handleMissingDependency(controller, platform, config)
|
||||
}
|
||||
|
||||
return RecordingResult.create({
|
||||
success: false,
|
||||
error: result.error || "",
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Error starting recording:", error)
|
||||
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred"
|
||||
|
||||
// Handle different error types
|
||||
if (errorMessage.includes("sign in")) {
|
||||
await handleSignInError(controller, errorMessage)
|
||||
} else {
|
||||
await showGenericError(errorMessage)
|
||||
}
|
||||
|
||||
return RecordingResult.create({
|
||||
success: false,
|
||||
error: errorMessage,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { telemetryService } from "@services/posthog/PostHogClientProvider"
|
||||
import { RecordedAudio } from "@shared/proto/cline/dictation"
|
||||
import { audioRecordingService } from "@/services/dictation/AudioRecordingService"
|
||||
import { Controller } from ".."
|
||||
|
||||
/**
|
||||
* Stops audio recording and returns the recorded audio
|
||||
* @param controller The controller instance
|
||||
* @returns RecordedAudio with audio data
|
||||
*/
|
||||
export const stopRecording = async (controller: Controller): Promise<RecordedAudio> => {
|
||||
const taskId = controller.task?.taskId
|
||||
const recordingStatus = audioRecordingService.getRecordingStatus()
|
||||
const recordingDuration = recordingStatus.durationSeconds * 1000 // Convert to milliseconds
|
||||
|
||||
try {
|
||||
const result = await audioRecordingService.stopRecording()
|
||||
|
||||
telemetryService.captureVoiceRecordingStopped(taskId, recordingDuration, result.success, process.platform)
|
||||
|
||||
return RecordedAudio.create({
|
||||
success: result.success,
|
||||
audioBase64: result.audioBase64 || "",
|
||||
error: result.error || "",
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Error stopping recording:", error)
|
||||
|
||||
telemetryService.captureVoiceRecordingStopped(taskId, recordingDuration, false, process.platform)
|
||||
|
||||
return RecordedAudio.create({
|
||||
success: false,
|
||||
audioBase64: "",
|
||||
error: error instanceof Error ? error.message : "Unknown error occurred",
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { telemetryService } from "@services/posthog/PostHogClientProvider"
|
||||
import { TranscribeAudioRequest, Transcription } from "@shared/proto/cline/dictation"
|
||||
import { HostProvider } from "@/hosts/host-provider"
|
||||
import { voiceTranscriptionService } from "@/services/dictation/VoiceTranscriptionService"
|
||||
import { ShowMessageType } from "@/shared/proto/host/window"
|
||||
import { Controller } from ".."
|
||||
|
||||
/**
|
||||
* Transcribes audio using Cline transcription service
|
||||
* @param controller The controller instance
|
||||
* @param request TranscribeAudioRequest containing base64 audio data
|
||||
* @returns Transcription with transcribed text or error
|
||||
*/
|
||||
export const transcribeAudio = async (controller: Controller, request: TranscribeAudioRequest): Promise<Transcription> => {
|
||||
const taskId = controller.task?.taskId
|
||||
const startTime = Date.now()
|
||||
|
||||
// Capture telemetry for transcription start
|
||||
telemetryService.captureVoiceTranscriptionStarted(taskId, request.language || "en")
|
||||
|
||||
try {
|
||||
// Transcribe the audio
|
||||
const result = await voiceTranscriptionService.transcribeAudio(request.audioBase64, request.language || "en")
|
||||
const durationMs = Date.now() - startTime
|
||||
|
||||
if (result.error) {
|
||||
let errorType = "api_error"
|
||||
if (result.error.includes("Authentication failed")) {
|
||||
errorType = "invalid_jwt_token"
|
||||
} else if (result.error.includes("Insufficient credits")) {
|
||||
errorType = "insufficient_credits"
|
||||
} else if (result.error.includes("Invalid audio format")) {
|
||||
errorType = "invalid_audio_format"
|
||||
} else if (result.error.includes("Cannot connect")) {
|
||||
errorType = "connection_error"
|
||||
} else if (result.error.includes("Network error")) {
|
||||
errorType = "network_error"
|
||||
}
|
||||
|
||||
telemetryService.captureVoiceTranscriptionError(taskId, errorType, result.error, durationMs)
|
||||
|
||||
let errorMessage = ""
|
||||
if (result.error.includes("Authentication failed")) {
|
||||
errorMessage = "Authentication failed. Please log in again."
|
||||
} else if (result.error.includes("Insufficient credits")) {
|
||||
errorMessage = "Insufficient credits for transcription service."
|
||||
} else if (result.error.includes("Cannot connect")) {
|
||||
errorMessage = "Cannot connect to transcription service."
|
||||
} else {
|
||||
errorMessage = `Voice transcription failed: ${result.error}`
|
||||
}
|
||||
|
||||
HostProvider.window.showMessage({
|
||||
type: ShowMessageType.ERROR,
|
||||
message: errorMessage,
|
||||
})
|
||||
} else if (result.text) {
|
||||
telemetryService.captureVoiceTranscriptionCompleted(taskId, result.text.length, durationMs, request.language || "en")
|
||||
}
|
||||
|
||||
return Transcription.create({
|
||||
text: result.text || "",
|
||||
error: result.error || "",
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Error transcribing audio:", error)
|
||||
const durationMs = Date.now() - startTime
|
||||
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred"
|
||||
|
||||
telemetryService.captureVoiceTranscriptionError(taskId, "unexpected_error", errorMessage, durationMs)
|
||||
|
||||
return Transcription.create({
|
||||
text: "",
|
||||
error: errorMessage,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -594,6 +594,7 @@ export class Controller {
|
||||
const taskHistory = this.stateManager.getGlobalStateKey("taskHistory")
|
||||
const autoApprovalSettings = this.stateManager.getGlobalStateKey("autoApprovalSettings")
|
||||
const browserSettings = this.stateManager.getGlobalStateKey("browserSettings")
|
||||
const dictationSettings = this.stateManager.getGlobalStateKey("dictationSettings")
|
||||
const focusChainSettings = this.stateManager.getGlobalStateKey("focusChainSettings")
|
||||
const focusChainFeatureFlagEnabled = this.stateManager.getGlobalStateKey("focusChainFeatureFlagEnabled")
|
||||
const preferredLanguage = this.stateManager.getGlobalStateKey("preferredLanguage")
|
||||
@@ -653,6 +654,7 @@ export class Controller {
|
||||
platform,
|
||||
autoApprovalSettings,
|
||||
browserSettings,
|
||||
dictationSettings,
|
||||
focusChainSettings,
|
||||
focusChainFeatureFlagEnabled,
|
||||
preferredLanguage,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { buildApiHandler } from "@core/api"
|
||||
import { telemetryService } from "@services/posthog/PostHogClientProvider"
|
||||
import { Empty } from "@shared/proto/cline/common"
|
||||
import {
|
||||
PlanActMode,
|
||||
@@ -7,10 +8,9 @@ import {
|
||||
UpdateSettingsRequest,
|
||||
} from "@shared/proto/cline/state"
|
||||
import { convertProtoApiConfigurationToApiConfiguration } from "@shared/proto-conversions/state/settings-conversion"
|
||||
import { OpenaiReasoningEffort } from "@shared/storage/types"
|
||||
import { TelemetrySetting } from "@shared/TelemetrySetting"
|
||||
import { McpDisplayMode } from "@/shared/McpDisplayMode"
|
||||
import { telemetryService } from "../../../services/posthog/PostHogClientProvider"
|
||||
import { OpenaiReasoningEffort } from "@/shared/storage/types"
|
||||
import { TelemetrySetting } from "@/shared/TelemetrySetting"
|
||||
import { Controller } from ".."
|
||||
|
||||
/**
|
||||
@@ -139,6 +139,10 @@ export async function updateSettings(controller: Controller, request: UpdateSett
|
||||
controller.stateManager.setGlobalState("strictPlanModeEnabled", request.strictPlanModeEnabled)
|
||||
}
|
||||
|
||||
if (request.dictationSettings !== undefined) {
|
||||
controller.stateManager.setGlobalState("dictationSettings", request.dictationSettings)
|
||||
}
|
||||
|
||||
// Update auto-condense setting
|
||||
if (request.useAutoCondense !== undefined) {
|
||||
if (controller.task) {
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { ApiConfiguration, fireworksDefaultModelId } from "@shared/api"
|
||||
import type { ExtensionContext } from "vscode"
|
||||
import { DEFAULT_AUTO_APPROVAL_SETTINGS } from "@/shared/AutoApprovalSettings"
|
||||
import { DEFAULT_DICTATION_SETTINGS } from "@/shared/DictationSettings"
|
||||
import { STATE_MANAGER_NOT_INITIALIZED } from "./error-messages"
|
||||
import { GlobalState, GlobalStateKey, LocalState, LocalStateKey, SecretKey, Secrets } from "./state-keys"
|
||||
import { readGlobalStateFromDisk, readSecretsFromDisk, readWorkspaceStateFromDisk } from "./utils/state-helpers"
|
||||
|
||||
@@ -4,6 +4,7 @@ import { LanguageModelChatSelector } from "vscode"
|
||||
import { AutoApprovalSettings } from "@/shared/AutoApprovalSettings"
|
||||
import { BrowserSettings } from "@/shared/BrowserSettings"
|
||||
import { ClineRulesToggles } from "@/shared/cline-rules"
|
||||
import { DictationSettings } from "@/shared/DictationSettings"
|
||||
import { HistoryItem } from "@/shared/HistoryItem"
|
||||
import { McpDisplayMode } from "@/shared/McpDisplayMode"
|
||||
import { McpMarketplaceCatalog } from "@/shared/mcp"
|
||||
@@ -79,6 +80,7 @@ export interface GlobalState {
|
||||
preferredLanguage: string
|
||||
openaiReasoningEffort: OpenaiReasoningEffort
|
||||
mode: Mode
|
||||
dictationSettings: DictationSettings
|
||||
focusChainSettings: FocusChainSettings
|
||||
focusChainFeatureFlagEnabled: boolean
|
||||
customPrompt: "compact" | undefined
|
||||
|
||||
@@ -4,6 +4,7 @@ import { Controller } from "@/core/controller"
|
||||
import { AutoApprovalSettings, DEFAULT_AUTO_APPROVAL_SETTINGS } from "@/shared/AutoApprovalSettings"
|
||||
import { BrowserSettings, DEFAULT_BROWSER_SETTINGS } from "@/shared/BrowserSettings"
|
||||
import { ClineRulesToggles } from "@/shared/cline-rules"
|
||||
import { DEFAULT_DICTATION_SETTINGS, DictationSettings } from "@/shared/DictationSettings"
|
||||
import { DEFAULT_FOCUS_CHAIN_SETTINGS, FocusChainSettings } from "@/shared/FocusChainSettings"
|
||||
import { HistoryItem } from "@/shared/HistoryItem"
|
||||
import { DEFAULT_MCP_DISPLAY_MODE, McpDisplayMode } from "@/shared/McpDisplayMode"
|
||||
@@ -201,9 +202,9 @@ export async function readGlobalStateFromDisk(context: ExtensionContext): Promis
|
||||
const difyBaseUrl = context.globalState.get("difyBaseUrl") as string | undefined
|
||||
const openaiReasoningEffort = context.globalState.get("openaiReasoningEffort") as OpenaiReasoningEffort | undefined
|
||||
const preferredLanguage = context.globalState.get("preferredLanguage") as string | undefined
|
||||
const dictationSettings = context.globalState.get("dictationSettings") as DictationSettings | undefined
|
||||
const focusChainSettings = context.globalState.get("focusChainSettings") as FocusChainSettings | undefined
|
||||
const focusChainFeatureFlagEnabled = context.globalState.get("focusChainFeatureFlagEnabled") as boolean | undefined
|
||||
|
||||
const mcpMarketplaceCatalog = context.globalState.get("mcpMarketplaceCatalog") as GlobalState["mcpMarketplaceCatalog"]
|
||||
const qwenCodeOauthPath = context.globalState.get("qwenCodeOauthPath") as GlobalState["qwenCodeOauthPath"]
|
||||
const customPrompt = context.globalState.get("customPrompt") as GlobalState["customPrompt"]
|
||||
@@ -422,6 +423,7 @@ export async function readGlobalStateFromDisk(context: ExtensionContext): Promis
|
||||
globalClineRulesToggles: globalClineRulesToggles || {},
|
||||
browserSettings: { ...DEFAULT_BROWSER_SETTINGS, ...browserSettings }, // this will ensure that older versions of browserSettings (e.g. before remoteBrowserEnabled was added) are merged with the default values (false for remoteBrowserEnabled)
|
||||
preferredLanguage: preferredLanguage || "English",
|
||||
dictationSettings: { ...DEFAULT_DICTATION_SETTINGS, ...dictationSettings },
|
||||
openaiReasoningEffort: (openaiReasoningEffort as OpenaiReasoningEffort) || "medium",
|
||||
mode: mode || "act",
|
||||
userInfo,
|
||||
|
||||
@@ -15,6 +15,7 @@ import * as vscode from "vscode"
|
||||
import { ToolUse, ToolUseName } from "../assistant-message"
|
||||
import { ContextManager } from "../context/context-management/ContextManager"
|
||||
import { formatResponse } from "../prompts/responses"
|
||||
import { ensureTaskDirectoryExists } from "../storage/disk"
|
||||
import { StateManager } from "../storage/StateManager"
|
||||
import { ToolResponse } from "."
|
||||
import { MessageStateHandler } from "./message-state"
|
||||
|
||||
@@ -2261,6 +2261,12 @@ export class Task {
|
||||
telemetryService.captureTaskInitialization(this.ulid, this.taskId, durationMs, this.enableCheckpoints)
|
||||
}
|
||||
|
||||
// Capture task initialization timing telemetry for the first API request
|
||||
if (isFirstRequest) {
|
||||
const durationMs = Math.round(performance.now() - this.taskInitializationStartTime)
|
||||
telemetryService.captureTaskInitialization(this.ulid, this.taskId, durationMs, this.enableCheckpoints)
|
||||
}
|
||||
|
||||
// since we sent off a placeholder api_req_started message to update the webview while waiting to actually start the API request (to load potential details for example), we need to update the text of that message
|
||||
const lastApiReqIndex = findLastIndex(this.messageStateHandler.getClineMessages(), (m) => m.say === "api_req_started")
|
||||
await this.messageStateHandler.updateClineMessage(lastApiReqIndex, {
|
||||
|
||||
@@ -240,4 +240,22 @@ export class ClineAccountService {
|
||||
await this._authService.restoreRefreshTokenAndRetrieveAuthInfo()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribes audio using the Cline transcription service
|
||||
* @param audioBase64 - Base64 encoded audio data
|
||||
* @param language - Optional language hint for transcription
|
||||
* @returns Promise with transcribed text or error
|
||||
*/
|
||||
async transcribeAudio(audioBase64: string, language?: string): Promise<{ text: string }> {
|
||||
const response = await this.authenticatedRequest<{ text: string }>(`/api/v1/chat/transcriptions`, {
|
||||
method: "POST",
|
||||
data: {
|
||||
audioData: audioBase64,
|
||||
language: language || "en",
|
||||
},
|
||||
})
|
||||
|
||||
return response
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
import { Logger } from "@services/logging/Logger"
|
||||
import { ChildProcess, spawn } from "child_process"
|
||||
import * as fs from "fs"
|
||||
import * as os from "os"
|
||||
import * as path from "path"
|
||||
import { AUDIO_PROGRAM_CONFIG } from "@/shared/audioProgramConstants"
|
||||
|
||||
function isExecutable(filePath: string): boolean {
|
||||
try {
|
||||
fs.accessSync(filePath, fs.constants.X_OK)
|
||||
return true
|
||||
} catch (e) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
export class AudioRecordingService {
|
||||
private recordingProcess: ChildProcess | null = null
|
||||
private startTime: number = 0
|
||||
private outputFile: string = ""
|
||||
|
||||
constructor() {}
|
||||
|
||||
/**
|
||||
* Determines if recording is currently active by checking process state
|
||||
*/
|
||||
private get isRecording(): boolean {
|
||||
return this.recordingProcess !== null && !this.recordingProcess.killed && this.recordingProcess.exitCode === null
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the recording state variables
|
||||
*/
|
||||
private resetRecordingState(): void {
|
||||
this.recordingProcess = null
|
||||
this.startTime = 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up the temporary audio file
|
||||
*/
|
||||
private async cleanupTempFile(): Promise<void> {
|
||||
if (this.outputFile && fs.existsSync(this.outputFile)) {
|
||||
try {
|
||||
fs.unlinkSync(this.outputFile)
|
||||
Logger.info("Temporary audio file cleaned up")
|
||||
} catch (error) {
|
||||
Logger.warn("Failed to cleanup temporary audio file: " + (error instanceof Error ? error.message : String(error)))
|
||||
} finally {
|
||||
this.outputFile = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminates the recording process gracefully
|
||||
*/
|
||||
private async terminateProcess(): Promise<void> {
|
||||
if (!this.recordingProcess) return
|
||||
|
||||
Logger.info("Terminating recording process...")
|
||||
this.recordingProcess.kill("SIGINT")
|
||||
|
||||
// Wait for the process to finish with timeout
|
||||
await new Promise<void>((resolve) => {
|
||||
const timeoutId = setTimeout(() => {
|
||||
Logger.warn("Process termination timed out after 5 seconds")
|
||||
resolve()
|
||||
}, 5000)
|
||||
|
||||
this.recordingProcess?.on("exit", (code) => {
|
||||
clearTimeout(timeoutId)
|
||||
Logger.info(`Recording process exited with code: ${code}`)
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs comprehensive cleanup of recording resources
|
||||
* @param options - Cleanup options
|
||||
* @param options.keepFile - If true, preserves the temporary file
|
||||
*/
|
||||
private async performCleanup(options?: { keepFile?: boolean }): Promise<void> {
|
||||
await this.terminateProcess()
|
||||
this.resetRecordingState()
|
||||
|
||||
if (!options?.keepFile) {
|
||||
await this.cleanupTempFile()
|
||||
}
|
||||
}
|
||||
|
||||
async startRecording(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
// Defensive cleanup before starting - ensures clean state
|
||||
if (this.recordingProcess || this.outputFile) {
|
||||
Logger.info("Performing pre-recording cleanup of stale resources...")
|
||||
await this.performCleanup()
|
||||
}
|
||||
|
||||
if (this.isRecording) {
|
||||
return { success: false, error: "Already recording" }
|
||||
}
|
||||
|
||||
// Check if recording software is available
|
||||
const checkResult = this.checkRecordingDependencies()
|
||||
if (!checkResult.available) {
|
||||
return { success: false, error: checkResult.error }
|
||||
}
|
||||
|
||||
// Create temporary file for audio output
|
||||
const tempDir = os.tmpdir()
|
||||
this.outputFile = path.join(tempDir, `cline_recording_${Date.now()}.webm`)
|
||||
|
||||
Logger.info("Starting audio recording...")
|
||||
|
||||
// Get the recording program path
|
||||
const recordProgram = this.getRecordProgram()
|
||||
if (!recordProgram) {
|
||||
return { success: false, error: "Recording program not found" }
|
||||
}
|
||||
Logger.info(`Using recording program: ${recordProgram.path}`)
|
||||
|
||||
// Set up recording arguments
|
||||
const args = recordProgram.getArgs(this.outputFile)
|
||||
|
||||
// Spawn the recording process
|
||||
this.recordingProcess = spawn(recordProgram.path, args)
|
||||
this.startTime = Date.now()
|
||||
|
||||
// Handle process errors
|
||||
this.recordingProcess.on("error", (error) => {
|
||||
Logger.error(`Recording process error: ${error.message}`)
|
||||
this.resetRecordingState()
|
||||
})
|
||||
|
||||
// Handle process exit
|
||||
this.recordingProcess.on("exit", (code) => {
|
||||
if (code !== 0 && code !== null) {
|
||||
Logger.warn(`Recording process exited with code: ${code}`)
|
||||
}
|
||||
})
|
||||
|
||||
this.recordingProcess.stderr?.on("data", (data) => {
|
||||
const message = data.toString().trim()
|
||||
if (message && !message.includes("In:") && !message.includes("Out:")) {
|
||||
Logger.info(`Recording stderr: ${message}`)
|
||||
}
|
||||
})
|
||||
|
||||
Logger.info("Audio recording started successfully")
|
||||
return { success: true }
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
Logger.error("Failed to start audio recording: " + errorMessage)
|
||||
return { success: false, error: `Failed to start recording: ${errorMessage}` }
|
||||
}
|
||||
}
|
||||
|
||||
async stopRecording(): Promise<{ success: boolean; audioBase64?: string; error?: string }> {
|
||||
try {
|
||||
if (!this.isRecording) {
|
||||
return { success: false, error: "Not currently recording" }
|
||||
}
|
||||
|
||||
Logger.info("Stopping audio recording...")
|
||||
|
||||
// Terminate the process but keep the file for reading
|
||||
await this.terminateProcess()
|
||||
this.resetRecordingState()
|
||||
|
||||
// Wait a moment for file to be fully written
|
||||
await new Promise((resolve) => setTimeout(resolve, 500))
|
||||
|
||||
// Read the audio file and convert to base64
|
||||
if (!fs.existsSync(this.outputFile)) {
|
||||
return { success: false, error: "Recording file not found" }
|
||||
}
|
||||
|
||||
const audioBuffer = fs.readFileSync(this.outputFile)
|
||||
const audioBase64 = audioBuffer.toString("base64")
|
||||
|
||||
// Clean up temporary file after reading
|
||||
await this.cleanupTempFile()
|
||||
|
||||
Logger.info("Audio recording stopped and converted to base64")
|
||||
return { success: true, audioBase64 }
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
Logger.error("Failed to stop audio recording: " + errorMessage)
|
||||
|
||||
// Ensure cleanup happens even on error
|
||||
await this.performCleanup()
|
||||
|
||||
return { success: false, error: `Failed to stop recording: ${errorMessage}` }
|
||||
}
|
||||
}
|
||||
|
||||
async cancelRecording(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
if (!this.isRecording) {
|
||||
return { success: false, error: "Not currently recording" }
|
||||
}
|
||||
|
||||
Logger.info("Canceling audio recording...")
|
||||
|
||||
// Perform full cleanup including file deletion
|
||||
await this.performCleanup()
|
||||
|
||||
Logger.info("Audio recording canceled successfully")
|
||||
return { success: true }
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
Logger.error("Failed to cancel audio recording: " + errorMessage)
|
||||
|
||||
// Ensure cleanup happens even on error
|
||||
await this.performCleanup()
|
||||
|
||||
return { success: false, error: `Failed to cancel recording: ${errorMessage}` }
|
||||
}
|
||||
}
|
||||
|
||||
getRecordingStatus(): { isRecording: boolean; durationSeconds: number; error?: string } {
|
||||
const durationSeconds = this.isRecording ? (Date.now() - this.startTime) / 1000 : 0
|
||||
return {
|
||||
isRecording: this.isRecording,
|
||||
durationSeconds,
|
||||
}
|
||||
}
|
||||
|
||||
private checkRecordingDependencies(): { available: boolean; error?: string } {
|
||||
const program = this.getRecordProgram()
|
||||
if (!program) {
|
||||
const platform = os.platform() as keyof typeof AUDIO_PROGRAM_CONFIG
|
||||
const config = AUDIO_PROGRAM_CONFIG[platform]
|
||||
const error = config ? config.error : `Audio recording is not supported on platform: ${platform}`
|
||||
return { available: false, error }
|
||||
}
|
||||
return { available: true }
|
||||
}
|
||||
|
||||
private getRecordProgram(): { path: string; getArgs: (outputFile: string) => string[] } | undefined {
|
||||
const platform = os.platform() as keyof typeof AUDIO_PROGRAM_CONFIG
|
||||
const config = AUDIO_PROGRAM_CONFIG[platform]
|
||||
|
||||
if (!config) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
// 1. Check if the command is in the system's PATH
|
||||
const pathDirs = (process.env.PATH || "").split(path.delimiter)
|
||||
for (const dir of pathDirs) {
|
||||
const fullPath = path.join(dir, config.command)
|
||||
if (fs.existsSync(fullPath) && isExecutable(fullPath)) {
|
||||
return { path: fullPath, getArgs: config.getArgs }
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Check fallback paths if not in PATH
|
||||
for (const p of config.fallbackPaths) {
|
||||
if (fs.existsSync(p) && isExecutable(p)) {
|
||||
return { path: p, getArgs: config.getArgs }
|
||||
}
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Public cleanup method for service shutdown
|
||||
*/
|
||||
cleanup(): void {
|
||||
// Use async cleanup but don't await since this is often called in sync contexts
|
||||
this.performCleanup().catch((error) => {
|
||||
Logger.error("Error during cleanup: " + (error instanceof Error ? error.message : String(error)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const audioRecordingService = new AudioRecordingService()
|
||||
@@ -0,0 +1,54 @@
|
||||
import { Logger } from "@services/logging/Logger"
|
||||
import axios from "axios"
|
||||
import { ClineAccountService } from "@/services/account/ClineAccountService"
|
||||
|
||||
export class VoiceTranscriptionService {
|
||||
private clineAccountService: ClineAccountService
|
||||
|
||||
constructor() {
|
||||
this.clineAccountService = ClineAccountService.getInstance()
|
||||
}
|
||||
|
||||
async transcribeAudio(audioBase64: string, language?: string): Promise<{ text?: string; error?: string }> {
|
||||
try {
|
||||
Logger.info("Transcribing audio with Cline transcription service...")
|
||||
|
||||
const result = await this.clineAccountService.transcribeAudio(audioBase64, language)
|
||||
|
||||
Logger.info("Transcription successful")
|
||||
|
||||
return { text: result.text }
|
||||
} catch (error) {
|
||||
Logger.error("Voice transcription error:", error)
|
||||
|
||||
// Handle axios errors with proper status code mapping
|
||||
if (axios.isAxiosError(error)) {
|
||||
const status = error.response?.status
|
||||
const message = error.response?.data?.message || error.message
|
||||
|
||||
switch (status) {
|
||||
case 401:
|
||||
return { error: "Authentication failed. Please reauthenticate your Cline account" }
|
||||
case 402:
|
||||
return { error: "Insufficient credits for transcription service." }
|
||||
case 400:
|
||||
return { error: "Invalid audio format or request data." }
|
||||
case 500:
|
||||
return { error: "Transcription server error. Please try again later." }
|
||||
default:
|
||||
return { error: `Transcription failed: ${message}` }
|
||||
}
|
||||
}
|
||||
|
||||
// Handle network errors
|
||||
const errorMessage = error instanceof Error ? error.message : String(error)
|
||||
if (errorMessage.includes("ECONNREFUSED") || errorMessage.includes("Network Error")) {
|
||||
return { error: "Cannot connect to transcription service." }
|
||||
}
|
||||
|
||||
return { error: `Network error: ${errorMessage}` }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const voiceTranscriptionService = new VoiceTranscriptionService()
|
||||
@@ -19,7 +19,7 @@ import type { PostHogClientProvider } from "../PostHogClientProvider"
|
||||
* When adding a new category, add it both here and to the initial values in telemetryCategoryEnabled
|
||||
* Ensure `if (!this.isCategoryEnabled('<category_name>')` is added to the capture method
|
||||
*/
|
||||
type TelemetryCategory = "checkpoints" | "browser" | "focus_chain"
|
||||
type TelemetryCategory = "checkpoints" | "browser" | "dictation" | "focus_chain"
|
||||
|
||||
/**
|
||||
* Maximum length for error messages to prevent excessive data
|
||||
@@ -31,6 +31,7 @@ export class TelemetryService {
|
||||
private telemetryCategoryEnabled: Map<TelemetryCategory, boolean> = new Map([
|
||||
["checkpoints", true], // Checkpoints telemetry enabled
|
||||
["browser", true], // Browser telemetry enabled
|
||||
["dictation", true], // Dictation telemetry enabled
|
||||
["focus_chain", true], // Focus Chain telemetry enabled
|
||||
])
|
||||
|
||||
@@ -120,6 +121,19 @@ export class TelemetryService {
|
||||
// Tracks when the rules menu button is clicked
|
||||
RULES_MENU_OPENED: "ui.rules_menu_opened",
|
||||
},
|
||||
DICTATION: {
|
||||
// Tracks when voice recording is started
|
||||
RECORDING_STARTED: "voice.recording_started",
|
||||
// Tracks when voice recording is stopped
|
||||
RECORDING_STOPPED: "voice.recording_stopped",
|
||||
// Tracks when voice transcription is started
|
||||
TRANSCRIPTION_STARTED: "voice.transcription_started",
|
||||
// Tracks when voice transcription is completed successfully
|
||||
TRANSCRIPTION_COMPLETED: "voice.transcription_completed",
|
||||
// Tracks when voice transcription fails
|
||||
TRANSCRIPTION_ERROR: "voice.transcription_error",
|
||||
// Tracks when voice feature is enabled or disabled in settings
|
||||
},
|
||||
}
|
||||
|
||||
/** Current version of the extension */
|
||||
@@ -676,6 +690,143 @@ export class TelemetryService {
|
||||
})
|
||||
}
|
||||
|
||||
// Dictation events
|
||||
/**
|
||||
* Records when voice recording is started
|
||||
* @param taskId Optional task identifier if recording was started during a task
|
||||
* @param platform The platform where recording is happening (macOS, Windows, Linux)
|
||||
* @param collect If true, collect event instead of sending
|
||||
*/
|
||||
public captureVoiceRecordingStarted(taskId?: string, platform?: string, collect: boolean = false) {
|
||||
if (!this.isCategoryEnabled("dictation")) {
|
||||
return
|
||||
}
|
||||
|
||||
this.capture({
|
||||
event: TelemetryService.EVENTS.DICTATION.RECORDING_STARTED,
|
||||
properties: {
|
||||
taskId,
|
||||
platform: platform || process.platform,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Records when voice recording is stopped
|
||||
* @param taskId Optional task identifier if recording was stopped during a task
|
||||
* @param durationMs Duration of the recording in milliseconds
|
||||
* @param success Whether the recording was successful
|
||||
* @param platform The platform where recording happened
|
||||
* @param collect If true, collect event instead of sending
|
||||
*/
|
||||
public captureVoiceRecordingStopped(
|
||||
taskId?: string,
|
||||
durationMs?: number,
|
||||
success?: boolean,
|
||||
platform?: string,
|
||||
collect?: boolean,
|
||||
) {
|
||||
if (!this.isCategoryEnabled("dictation")) {
|
||||
return
|
||||
}
|
||||
|
||||
this.capture({
|
||||
event: TelemetryService.EVENTS.DICTATION.RECORDING_STOPPED,
|
||||
properties: {
|
||||
taskId,
|
||||
durationMs,
|
||||
success,
|
||||
platform: platform || process.platform,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Records when voice transcription is started
|
||||
* @param taskId Optional task identifier if transcription was started during a task
|
||||
* @param audioSizeBytes Size of the audio data being transcribed
|
||||
* @param language Language hint provided for transcription
|
||||
* @param collect If true, collect event instead of sending
|
||||
*/
|
||||
public captureVoiceTranscriptionStarted(taskId?: string, language?: string, collect: boolean = false) {
|
||||
if (!this.isCategoryEnabled("dictation")) {
|
||||
return
|
||||
}
|
||||
|
||||
this.capture({
|
||||
event: TelemetryService.EVENTS.DICTATION.TRANSCRIPTION_STARTED,
|
||||
properties: {
|
||||
taskId,
|
||||
language,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Records when voice transcription is completed successfully
|
||||
* @param taskId Optional task identifier if transcription was completed during a task
|
||||
* @param transcriptionLength Length of the transcribed text
|
||||
* @param durationMs Time taken for transcription in milliseconds
|
||||
* @param language Language used for transcription
|
||||
* @param collect If true, collect event instead of sending
|
||||
*/
|
||||
public captureVoiceTranscriptionCompleted(
|
||||
taskId?: string,
|
||||
transcriptionLength?: number,
|
||||
durationMs?: number,
|
||||
language?: string,
|
||||
collect: boolean = false,
|
||||
) {
|
||||
if (!this.isCategoryEnabled("dictation")) {
|
||||
return
|
||||
}
|
||||
|
||||
this.capture({
|
||||
event: TelemetryService.EVENTS.DICTATION.TRANSCRIPTION_COMPLETED,
|
||||
properties: {
|
||||
taskId,
|
||||
transcriptionLength,
|
||||
durationMs,
|
||||
language,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Records when voice transcription fails
|
||||
* @param taskId Optional task identifier if transcription failed during a task
|
||||
* @param errorType Type of error that occurred (e.g., "no_openai_key", "api_error", "network_error")
|
||||
* @param errorMessage The error message
|
||||
* @param durationMs Time taken before failure in milliseconds
|
||||
* @param collect If true, collect event instead of sending
|
||||
*/
|
||||
public captureVoiceTranscriptionError(
|
||||
taskId?: string,
|
||||
errorType?: string,
|
||||
errorMessage?: string,
|
||||
durationMs?: number,
|
||||
collect: boolean = false,
|
||||
) {
|
||||
if (!this.isCategoryEnabled("dictation")) {
|
||||
return
|
||||
}
|
||||
|
||||
this.capture({
|
||||
event: TelemetryService.EVENTS.DICTATION.TRANSCRIPTION_ERROR,
|
||||
properties: {
|
||||
taskId,
|
||||
errorType,
|
||||
errorMessage,
|
||||
durationMs,
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Records when focus chain is enabled/disabled by the user
|
||||
* @param enabled Whether focus chain was enabled (true) or disabled (false)
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
export interface DictationSettings {
|
||||
dictationEnabled: boolean
|
||||
dictationLanguage: string
|
||||
}
|
||||
|
||||
export const DEFAULT_DICTATION_SETTINGS: DictationSettings = {
|
||||
dictationEnabled: false, // While this service is in Experimental status, we should default to false
|
||||
dictationLanguage: "en",
|
||||
}
|
||||
|
||||
export interface LanguageItem {
|
||||
name: string
|
||||
code: string
|
||||
}
|
||||
|
||||
export const SUPPORTED_DICTATION_LANGUAGES: LanguageItem[] = [
|
||||
{ name: "English", code: "en" },
|
||||
{ name: "Spanish (Español)", code: "es" },
|
||||
{ name: "Chinese (中文)", code: "zh" },
|
||||
{ name: "Japanese (日本語)", code: "ja" },
|
||||
{ name: "Afrikaans", code: "af" },
|
||||
{ name: "Arabic (العربية)", code: "ar" },
|
||||
{ name: "Armenian (Հայերեն)", code: "hy" },
|
||||
{ name: "Azerbaijani (Azərbaycan)", code: "az" },
|
||||
{ name: "Belarusian (Беларуская)", code: "be" },
|
||||
{ name: "Bosnian (Bosanski)", code: "bs" },
|
||||
{ name: "Bulgarian (Български)", code: "bg" },
|
||||
{ name: "Catalan (Català)", code: "ca" },
|
||||
{ name: "Croatian (Hrvatski)", code: "hr" },
|
||||
{ name: "Czech (Čeština)", code: "cs" },
|
||||
{ name: "Danish (Dansk)", code: "da" },
|
||||
{ name: "Dutch (Nederlands)", code: "nl" },
|
||||
{ name: "Estonian (Eesti)", code: "et" },
|
||||
{ name: "Finnish (Suomi)", code: "fi" },
|
||||
{ name: "French (Français)", code: "fr" },
|
||||
{ name: "Galician (Galego)", code: "gl" },
|
||||
{ name: "German (Deutsch)", code: "de" },
|
||||
{ name: "Greek (Ελληνικά)", code: "el" },
|
||||
{ name: "Hebrew (עברית)", code: "he" },
|
||||
{ name: "Hindi (हिन्दी)", code: "hi" },
|
||||
{ name: "Hungarian (Magyar)", code: "hu" },
|
||||
{ name: "Icelandic (Íslenska)", code: "is" },
|
||||
{ name: "Indonesian (Bahasa Indonesia)", code: "id" },
|
||||
{ name: "Italian (Italiano)", code: "it" },
|
||||
{ name: "Kannada (ಕನ್ನಡ)", code: "kn" },
|
||||
{ name: "Kazakh (Қазақша)", code: "kk" },
|
||||
{ name: "Korean (한국어)", code: "ko" },
|
||||
{ name: "Latvian (Latviešu)", code: "lv" },
|
||||
{ name: "Lithuanian (Lietuvių)", code: "lt" },
|
||||
{ name: "Macedonian (Македонски)", code: "mk" },
|
||||
{ name: "Malay (Bahasa Melayu)", code: "ms" },
|
||||
{ name: "Marathi (मराठी)", code: "mr" },
|
||||
{ name: "Maori (Te Reo Māori)", code: "mi" },
|
||||
{ name: "Nepali (नेपाली)", code: "ne" },
|
||||
{ name: "Norwegian (Norsk)", code: "no" },
|
||||
{ name: "Persian (فارسی)", code: "fa" },
|
||||
{ name: "Polish (Polski)", code: "pl" },
|
||||
{ name: "Portuguese (Português)", code: "pt" },
|
||||
{ name: "Romanian (Română)", code: "ro" },
|
||||
{ name: "Russian (Русский)", code: "ru" },
|
||||
{ name: "Serbian (Српски)", code: "sr" },
|
||||
{ name: "Slovak (Slovenčina)", code: "sk" },
|
||||
{ name: "Slovenian (Slovenščina)", code: "sl" },
|
||||
{ name: "Swahili (Kiswahili)", code: "sw" },
|
||||
{ name: "Swedish (Svenska)", code: "sv" },
|
||||
{ name: "Tagalog", code: "tl" },
|
||||
{ name: "Tamil (தமிழ்)", code: "ta" },
|
||||
{ name: "Thai (ไทย)", code: "th" },
|
||||
{ name: "Turkish (Türkçe)", code: "tr" },
|
||||
{ name: "Ukrainian (Українська)", code: "uk" },
|
||||
{ name: "Urdu (اردو)", code: "ur" },
|
||||
{ name: "Vietnamese (Tiếng Việt)", code: "vi" },
|
||||
{ name: "Welsh (Cymraeg)", code: "cy" },
|
||||
]
|
||||
@@ -4,6 +4,7 @@ import { AutoApprovalSettings } from "./AutoApprovalSettings"
|
||||
import { ApiConfiguration } from "./api"
|
||||
import { BrowserSettings } from "./BrowserSettings"
|
||||
import { ClineRulesToggles } from "./cline-rules"
|
||||
import { DictationSettings } from "./DictationSettings"
|
||||
import { FocusChainSettings } from "./FocusChainSettings"
|
||||
import { HistoryItem } from "./HistoryItem"
|
||||
import { McpDisplayMode } from "./McpDisplayMode"
|
||||
@@ -35,6 +36,7 @@ export interface ExtensionState {
|
||||
apiConfiguration?: ApiConfiguration
|
||||
autoApprovalSettings: AutoApprovalSettings
|
||||
browserSettings: BrowserSettings
|
||||
dictationSettings: DictationSettings
|
||||
remoteBrowserHost?: string
|
||||
preferredLanguage?: string
|
||||
openaiReasoningEffort?: OpenaiReasoningEffort
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
export const AUDIO_PROGRAM_CONFIG = {
|
||||
darwin: {
|
||||
command: "ffmpeg",
|
||||
fallbackPaths: ["/usr/local/bin/ffmpeg", "/opt/homebrew/bin/ffmpeg"],
|
||||
getArgs: (outputFile: string) => [
|
||||
"-f",
|
||||
"avfoundation",
|
||||
"-i",
|
||||
":default",
|
||||
"-c:a",
|
||||
"libopus",
|
||||
"-b:a",
|
||||
"32k",
|
||||
"-application",
|
||||
"voip",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-ac",
|
||||
"1",
|
||||
outputFile,
|
||||
],
|
||||
dependencyName: "FFmpeg",
|
||||
installCommand: "brew install ffmpeg",
|
||||
error: "FFmpeg is required for voice recording but is not installed on your system.",
|
||||
installDescription: "FFmpeg is a multimedia framework that Cline uses to record audio from your microphone.",
|
||||
},
|
||||
linux: {
|
||||
command: "ffmpeg",
|
||||
fallbackPaths: ["/usr/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/snap/bin/ffmpeg"],
|
||||
getArgs: (outputFile: string) => [
|
||||
"-f",
|
||||
"alsa",
|
||||
"-i",
|
||||
"default",
|
||||
"-c:a",
|
||||
"libopus",
|
||||
"-b:a",
|
||||
"32k",
|
||||
"-application",
|
||||
"voip",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-ac",
|
||||
"1",
|
||||
outputFile,
|
||||
],
|
||||
dependencyName: "FFmpeg",
|
||||
installCommand: "sudo apt-get update && sudo apt-get install -y ffmpeg",
|
||||
error: "FFmpeg is required for voice recording but is not installed on your system.",
|
||||
installDescription: "FFmpeg is a multimedia framework that Cline uses to record audio from your microphone.",
|
||||
},
|
||||
win32: {
|
||||
command: "ffmpeg",
|
||||
fallbackPaths: [
|
||||
"C:\\ffmpeg\\bin\\ffmpeg.exe",
|
||||
"C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe",
|
||||
"C:\\Program Files (x86)\\ffmpeg\\bin\\ffmpeg.exe",
|
||||
],
|
||||
getArgs: (outputFile: string) => [
|
||||
"-f",
|
||||
"wasapi",
|
||||
"-i",
|
||||
"audio=default",
|
||||
"-c:a",
|
||||
"libopus",
|
||||
"-b:a",
|
||||
"32k",
|
||||
"-application",
|
||||
"voip",
|
||||
"-ar",
|
||||
"16000",
|
||||
"-ac",
|
||||
"1",
|
||||
outputFile,
|
||||
],
|
||||
dependencyName: "FFmpeg",
|
||||
installCommand: "winget install Gyan.FFmpeg",
|
||||
error: "FFmpeg is required for voice recording but is not installed on your system.",
|
||||
installDescription: "FFmpeg is a multimedia framework that Cline uses to record audio from your microphone.",
|
||||
},
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
import { ApiConfiguration, ApiProvider, BedrockModelId } from "@shared/api"
|
||||
import { ApiConfiguration as ProtoApiConfiguration } from "@shared/proto/cline/state"
|
||||
|
||||
/**
|
||||
* Domain -> Proto conversions
|
||||
*/
|
||||
|
||||
/**
|
||||
* Converts domain ApiConfiguration objects to proto ApiConfiguration objects
|
||||
*/
|
||||
@@ -142,6 +146,10 @@ export function convertApiConfigurationToProtoApiConfiguration(config: ApiConfig
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Proto -> Domain conversions
|
||||
*/
|
||||
|
||||
/**
|
||||
* Converts proto ApiConfiguration objects to domain ApiConfiguration objects
|
||||
*/
|
||||
|
||||
Generated
+1525
-2201
File diff suppressed because it is too large
Load Diff
@@ -17,7 +17,8 @@
|
||||
"dependencies": {
|
||||
"@floating-ui/react": "^0.27.4",
|
||||
"@fontsource/azeret-mono": "^5.2.9",
|
||||
"@heroui/react": "^2.8.0-beta.2",
|
||||
"@heroui/react": "^2.8.2",
|
||||
"@paper-design/shaders-react": "^0.0.46",
|
||||
"@vscode/webview-ui-toolkit": "^1.4.0",
|
||||
"debounce": "^2.1.1",
|
||||
"dompurify": "^3.2.4",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { PulsingBorder } from "@paper-design/shaders-react"
|
||||
import { mentionRegex, mentionRegexGlobal } from "@shared/context-mentions"
|
||||
import { EmptyRequest, StringRequest } from "@shared/proto/cline/common"
|
||||
import { FileSearchRequest, FileSearchType, RelativePathsRequest } from "@shared/proto/cline/file"
|
||||
@@ -45,6 +46,7 @@ import {
|
||||
import { validateApiConfiguration, validateModelId } from "@/utils/validate"
|
||||
import ClineRulesToggleModal from "../cline-rules/ClineRulesToggleModal"
|
||||
import ServersToggleModal from "./ServersToggleModal"
|
||||
import VoiceRecorder from "./VoiceRecorder"
|
||||
|
||||
const { MAX_IMAGES_AND_FILES_PER_MESSAGE } = CHAT_CONSTANTS
|
||||
|
||||
@@ -279,8 +281,15 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
},
|
||||
ref,
|
||||
) => {
|
||||
const { mode, apiConfiguration, openRouterModels, platform, localWorkflowToggles, globalWorkflowToggles } =
|
||||
useExtensionState()
|
||||
const {
|
||||
mode,
|
||||
apiConfiguration,
|
||||
openRouterModels,
|
||||
platform,
|
||||
localWorkflowToggles,
|
||||
globalWorkflowToggles,
|
||||
dictationSettings,
|
||||
} = useExtensionState()
|
||||
const [isTextAreaFocused, setIsTextAreaFocused] = useState(false)
|
||||
const [isDraggingOver, setIsDraggingOver] = useState(false)
|
||||
const [gitCommits, setGitCommits] = useState<GitCommit[]>([])
|
||||
@@ -317,6 +326,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
const unsupportedFileTimerRef = useRef<NodeJS.Timeout | null>(null)
|
||||
const [showDimensionError, setShowDimensionError] = useState(false)
|
||||
const dimensionErrorTimerRef = useRef<NodeJS.Timeout | null>(null)
|
||||
const [isVoiceRecording, setIsVoiceRecording] = useState(false)
|
||||
|
||||
const [fileSearchResults, setFileSearchResults] = useState<SearchResult[]>([])
|
||||
const [searchLoading, setSearchLoading] = useState(false)
|
||||
@@ -1410,6 +1420,11 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
)
|
||||
}
|
||||
|
||||
const handleSetVoiceRecording = (isRecording: boolean) => {
|
||||
setIsVoiceRecording(isRecording)
|
||||
sendingDisabled = isRecording
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div
|
||||
@@ -1425,6 +1440,44 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
// Drag-over styles moved to DynamicTextArea
|
||||
transition: "background-color 0.1s ease-in-out, border 0.1s ease-in-out",
|
||||
}}>
|
||||
<div
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: "10px 15px", // match textarea/hightlight inset
|
||||
pointerEvents: "none",
|
||||
zIndex: 1, // Above textarea but below menus
|
||||
overflow: "hidden", // clip shader to rounded rect
|
||||
borderRadius: 2, // match textarea radius
|
||||
transition: "opacity 1s ease-in-out",
|
||||
opacity: isVoiceRecording ? 1 : 0,
|
||||
}}>
|
||||
<PulsingBorder
|
||||
bloom={1}
|
||||
colorBack={"rgba(0,0,0,0)"}
|
||||
colors={[
|
||||
"#ffffff", // white
|
||||
"#ffffff",
|
||||
"#9d57fa",
|
||||
"#ffffff",
|
||||
]} // Match textarea border radius
|
||||
intensity={0.97}
|
||||
pulse={0}
|
||||
roundness={0}
|
||||
scale={1.0}
|
||||
smoke={0.18}
|
||||
smokeSize={0.76}
|
||||
softness={1}
|
||||
speed={1}
|
||||
spotSize={0.4}
|
||||
spots={3}
|
||||
style={{
|
||||
width: "100%",
|
||||
height: "100%",
|
||||
}}
|
||||
thickness={0}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{showDimensionError && (
|
||||
<div
|
||||
style={{
|
||||
@@ -1508,7 +1561,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
style={{
|
||||
position: "absolute",
|
||||
inset: "10px 15px",
|
||||
border: "1px solid var(--vscode-input-border)",
|
||||
// border: "1px solid var(--vscode-input-border)",
|
||||
borderRadius: 2,
|
||||
pointerEvents: "none",
|
||||
zIndex: 5,
|
||||
@@ -1601,7 +1654,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
// borderLeft: "9px solid transparent", // NOTE: react-textarea-autosize doesn't calculate correct height when using borderLeft/borderRight so we need to use horizontal padding instead
|
||||
// Instead of using boxShadow, we use a div with a border to better replicate the behavior when the textarea is focused
|
||||
// boxShadow: "0px 0px 0px 1px var(--vscode-input-border)",
|
||||
padding: "9px 28px 9px 9px",
|
||||
padding: `9px ${dictationSettings?.dictationEnabled ? "48" : "28"}px 9px 9px`,
|
||||
cursor: "text",
|
||||
flex: 1,
|
||||
zIndex: 1,
|
||||
@@ -1654,6 +1707,43 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
flexDirection: "row",
|
||||
alignItems: "center",
|
||||
}}>
|
||||
{dictationSettings?.dictationEnabled === true && (
|
||||
<VoiceRecorder
|
||||
disabled={sendingDisabled}
|
||||
language={dictationSettings?.dictationLanguage || "en"}
|
||||
onProcessingStateChange={(isProcessing, message) => {
|
||||
if (isProcessing && message) {
|
||||
// Show processing message in input
|
||||
const processingText = inputValue + (inputValue ? " " : "") + `[${message}]`
|
||||
setInputValue(processingText)
|
||||
}
|
||||
// When processing is done, the onTranscription callback will handle the final text
|
||||
}}
|
||||
onRecordingStateChange={handleSetVoiceRecording}
|
||||
onTranscription={(text) => {
|
||||
// Remove any processing text first
|
||||
const processingPattern = /\s*\[Transcribing\.\.\.\]$/
|
||||
const cleanedValue = inputValue.replace(processingPattern, "")
|
||||
|
||||
if (!text) {
|
||||
setInputValue(cleanedValue)
|
||||
return
|
||||
}
|
||||
|
||||
// Append the transcribed text to the cleaned input
|
||||
const newValue = cleanedValue + (cleanedValue ? " " : "") + text
|
||||
setInputValue(newValue)
|
||||
// Focus the textarea and move cursor to end
|
||||
setTimeout(() => {
|
||||
if (textAreaRef.current) {
|
||||
textAreaRef.current.focus()
|
||||
const length = newValue.length
|
||||
textAreaRef.current.setSelectionRange(length, length)
|
||||
}
|
||||
}, 0)
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
{/* <div
|
||||
className={`input-icon-button ${shouldDisableImages ? "disabled" : ""} codicon codicon-device-camera`}
|
||||
onClick={() => {
|
||||
@@ -1666,16 +1756,18 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
|
||||
fontSize: 16.5,
|
||||
}}
|
||||
/> */}
|
||||
<div
|
||||
className={`input-icon-button ${sendingDisabled ? "disabled" : ""} codicon codicon-send`}
|
||||
data-testid="send-button"
|
||||
onClick={() => {
|
||||
if (!sendingDisabled) {
|
||||
setIsTextAreaFocused(false)
|
||||
onSend()
|
||||
}
|
||||
}}
|
||||
style={{ fontSize: 15 }}></div>
|
||||
{!isVoiceRecording && (
|
||||
<div
|
||||
className={`input-icon-button ${sendingDisabled ? "disabled" : ""} codicon codicon-send`}
|
||||
data-testid="send-button"
|
||||
onClick={() => {
|
||||
if (!sendingDisabled) {
|
||||
setIsTextAreaFocused(false)
|
||||
onSend()
|
||||
}
|
||||
}}
|
||||
style={{ fontSize: 15 }}></div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { TranscribeAudioRequest } from "@shared/proto/cline/dictation"
|
||||
import { EmptyRequest } from "@shared/proto/index.cline"
|
||||
import React, { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { DictationServiceClient } from "@/services/grpc-client"
|
||||
import { formatSeconds } from "@/utils/format"
|
||||
import HeroTooltip from "../common/HeroTooltip"
|
||||
|
||||
interface VoiceRecorderProps {
|
||||
onTranscription: (text: string) => void
|
||||
onProcessingStateChange?: (isProcessing: boolean, message?: string) => void
|
||||
onRecordingStateChange?: (isRecording: boolean) => void
|
||||
disabled?: boolean
|
||||
language?: string
|
||||
}
|
||||
|
||||
const MAX_DURATION = 5 * 60 // 5 minutes in seconds
|
||||
|
||||
const VoiceRecorder: React.FC<VoiceRecorderProps> = ({
|
||||
onTranscription,
|
||||
onProcessingStateChange,
|
||||
onRecordingStateChange,
|
||||
disabled = false,
|
||||
language = "en",
|
||||
}) => {
|
||||
const [isRecording, setIsRecording] = useState(false)
|
||||
const [isProcessing, setIsProcessing] = useState(false)
|
||||
const [recordingDuration, setRecordingDuration] = useState(0)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const pollingIntervalRef = useRef<NodeJS.Timeout | null>(null)
|
||||
|
||||
// Notify parent when recording state changes
|
||||
useEffect(() => {
|
||||
onRecordingStateChange?.(isRecording)
|
||||
}, [isRecording, onRecordingStateChange])
|
||||
|
||||
const startRecording = useCallback(async () => {
|
||||
try {
|
||||
setIsRecording(true)
|
||||
setError(null) // Clear any previous errors
|
||||
onProcessingStateChange?.(false) // Clear any previous processing state
|
||||
setRecordingDuration(0) // Reset recording duration
|
||||
|
||||
// Call Extension Host to start recording
|
||||
const response = await DictationServiceClient.startRecording(EmptyRequest.create({}))
|
||||
|
||||
if (!response.success) {
|
||||
console.error("Failed to start recording:", response.error)
|
||||
setIsRecording(false)
|
||||
setError(response.error || "Failed to start recording")
|
||||
return
|
||||
}
|
||||
|
||||
console.log("Recording started successfully")
|
||||
} catch (error) {
|
||||
console.error("Error starting recording:", error)
|
||||
setIsRecording(false)
|
||||
const errorMessage = error instanceof Error ? error.message : "Failed to start recording"
|
||||
setError(errorMessage)
|
||||
}
|
||||
}, [onProcessingStateChange])
|
||||
|
||||
const stopRecording = useCallback(async () => {
|
||||
try {
|
||||
setIsRecording(false)
|
||||
setIsProcessing(true)
|
||||
onProcessingStateChange?.(true, "Processing...")
|
||||
|
||||
// Call Extension Host to stop recording and get audio
|
||||
const response = await DictationServiceClient.stopRecording(EmptyRequest.create({}))
|
||||
|
||||
if (!response.success) {
|
||||
console.error("Failed to stop recording:", response.error)
|
||||
setIsProcessing(false)
|
||||
const errorMessage = response.error || "Failed to stop recording"
|
||||
setError(errorMessage)
|
||||
onTranscription("")
|
||||
return
|
||||
}
|
||||
|
||||
if (!response.audioBase64) {
|
||||
console.error("No audio data received")
|
||||
setIsProcessing(false)
|
||||
const errorMessage = "No audio data received"
|
||||
setError(errorMessage)
|
||||
onTranscription("")
|
||||
return
|
||||
}
|
||||
|
||||
// Update processing state for transcription
|
||||
onProcessingStateChange?.(true, "Transcribing...")
|
||||
|
||||
// Transcribe the audio using OpenAI Whisper
|
||||
const transcriptionResponse = await DictationServiceClient.transcribeAudio(
|
||||
TranscribeAudioRequest.create({
|
||||
audioBase64: response.audioBase64,
|
||||
language: language,
|
||||
}),
|
||||
)
|
||||
|
||||
if (transcriptionResponse.error) {
|
||||
console.error("Transcription error:", transcriptionResponse.error)
|
||||
setError(transcriptionResponse.error)
|
||||
onTranscription("")
|
||||
// Clear the error after a delay
|
||||
setTimeout(() => {
|
||||
setError(null)
|
||||
onProcessingStateChange?.(false)
|
||||
}, 5000)
|
||||
} else if (transcriptionResponse.text) {
|
||||
setError(null)
|
||||
onTranscription(transcriptionResponse.text)
|
||||
onProcessingStateChange?.(false)
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error stopping recording:", error)
|
||||
const errorMessage = error instanceof Error ? error.message : "An error occurred"
|
||||
setError(errorMessage)
|
||||
onTranscription("")
|
||||
} finally {
|
||||
setIsProcessing(false)
|
||||
}
|
||||
}, [onTranscription, onProcessingStateChange])
|
||||
|
||||
// Poll recording status while recording to update duration
|
||||
useEffect(() => {
|
||||
const pollRecordingStatus = async () => {
|
||||
try {
|
||||
const statusResponse = await DictationServiceClient.getRecordingStatus(EmptyRequest.create({}))
|
||||
if (statusResponse.isRecording) {
|
||||
setRecordingDuration(Math.floor(statusResponse.durationSeconds))
|
||||
|
||||
// Auto-stop if max duration reached
|
||||
if (statusResponse.durationSeconds >= MAX_DURATION) {
|
||||
stopRecording()
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error polling recording status:", error)
|
||||
}
|
||||
}
|
||||
|
||||
if (isRecording && !isProcessing) {
|
||||
pollingIntervalRef.current = setInterval(pollRecordingStatus, 1000)
|
||||
} else {
|
||||
// Clear polling when not recording
|
||||
if (pollingIntervalRef.current) {
|
||||
clearInterval(pollingIntervalRef.current)
|
||||
pollingIntervalRef.current = null
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup on unmount
|
||||
return () => {
|
||||
if (pollingIntervalRef.current) {
|
||||
clearInterval(pollingIntervalRef.current)
|
||||
pollingIntervalRef.current = null
|
||||
}
|
||||
}
|
||||
}, [isRecording, isProcessing, stopRecording])
|
||||
|
||||
const cancelRecording = useCallback(async () => {
|
||||
try {
|
||||
setIsRecording(false)
|
||||
setError(null)
|
||||
onProcessingStateChange?.(false)
|
||||
onTranscription("")
|
||||
|
||||
// Call Extension Host to cancel recording
|
||||
const response = await DictationServiceClient.cancelRecording(EmptyRequest.create({}))
|
||||
|
||||
if (!response.success) {
|
||||
console.error("Failed to cancel recording:", response.error)
|
||||
setError(response.error || "Failed to cancel recording")
|
||||
return
|
||||
}
|
||||
|
||||
console.log("Recording canceled successfully")
|
||||
} catch (error) {
|
||||
console.error("Error canceling recording:", error)
|
||||
const errorMessage = error instanceof Error ? error.message : "Failed to cancel recording"
|
||||
setError(errorMessage)
|
||||
}
|
||||
}, [onProcessingStateChange, onTranscription])
|
||||
|
||||
const handleStartClick = useCallback(() => {
|
||||
if (disabled || isProcessing) return
|
||||
if (error) return setError(null)
|
||||
startRecording()
|
||||
}, [startRecording, disabled, isProcessing, error])
|
||||
|
||||
const handleCancelClick = useCallback(() => {
|
||||
if (disabled || isProcessing) return
|
||||
cancelRecording()
|
||||
}, [cancelRecording, disabled, isProcessing])
|
||||
|
||||
const handleStopClick = useCallback(() => {
|
||||
if (disabled || isProcessing) return
|
||||
stopRecording()
|
||||
}, [stopRecording, disabled, isProcessing])
|
||||
|
||||
// When not recording, show single mic button
|
||||
if (!isRecording) {
|
||||
const iconClass = isProcessing ? "codicon-loading" : error ? "codicon-error" : "codicon-mic"
|
||||
const iconColor = error ? "var(--vscode-errorForeground)" : ""
|
||||
const iconAnimation = isProcessing ? "animate-spin" : ""
|
||||
const iconAdjustment = isProcessing ? "mt-0" : error ? "mt-1" : "mt-0.5"
|
||||
const tooltipContent = isProcessing ? "Transcribing..." : error ? `Error: ${error}` : null
|
||||
|
||||
return (
|
||||
<HeroTooltip content={tooltipContent} placement="top">
|
||||
<div
|
||||
className={`input-icon-button mr-1.5 text-base ${iconAdjustment} ${iconAnimation} ${disabled || isProcessing ? "disabled" : ""}`}
|
||||
onClick={handleStartClick}
|
||||
style={{ color: iconColor }}>
|
||||
<span className={`codicon ${iconClass}`} />
|
||||
</div>
|
||||
</HeroTooltip>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={`flex items-center ${isRecording ? "mr-0.5" : "mr-1.5"}`}>
|
||||
<HeroTooltip
|
||||
content={`Stop Recording (${formatSeconds(recordingDuration)}/${formatSeconds(MAX_DURATION)})`}
|
||||
placement="top">
|
||||
<div
|
||||
className={`input-icon-button text-base mr-1 mt-1 animate-pulse text-[var(--vscode-errorForeground)] ${disabled || isProcessing ? "disabled" : ""}`}
|
||||
onClick={handleStopClick}>
|
||||
<span className="codicon codicon-stop-circle" />
|
||||
</div>
|
||||
</HeroTooltip>
|
||||
<HeroTooltip content="Cancel Recording" placement="top">
|
||||
<div
|
||||
className={`input-icon-button text-base mt-1 text-[var(--vscode-textForeground)] ${disabled || isProcessing ? "disabled" : ""}`}
|
||||
onClick={handleCancelClick}>
|
||||
<span className="codicon codicon-close" />
|
||||
</div>
|
||||
</HeroTooltip>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default VoiceRecorder
|
||||
@@ -0,0 +1,16 @@
|
||||
import styled from "styled-components"
|
||||
|
||||
const CollapsibleContent = styled.div<{ isOpen: boolean }>`
|
||||
overflow: hidden;
|
||||
transition:
|
||||
max-height 0.3s ease-in-out,
|
||||
opacity 0.3s ease-in-out,
|
||||
margin-top 0.3s ease-in-out,
|
||||
visibility 0.3s ease-in-out;
|
||||
max-height: ${({ isOpen }) => (isOpen ? "1000px" : "0")};
|
||||
opacity: ${({ isOpen }) => (isOpen ? 1 : 0)};
|
||||
margin-top: ${({ isOpen }) => (isOpen ? "15px" : "0")};
|
||||
visibility: ${({ isOpen }) => (isOpen ? "visible" : "hidden")};
|
||||
`
|
||||
|
||||
export default CollapsibleContent
|
||||
@@ -5,6 +5,7 @@ import styled from "styled-components"
|
||||
import { BROWSER_VIEWPORT_PRESETS } from "../../../../../src/shared/BrowserSettings"
|
||||
import { useExtensionState } from "../../../context/ExtensionStateContext"
|
||||
import { BrowserServiceClient } from "../../../services/grpc-client"
|
||||
import CollapsibleContent from "../CollapsibleContent"
|
||||
import { DebouncedTextField } from "../common/DebouncedTextField"
|
||||
import Section from "../Section"
|
||||
import { updateBrowserSetting } from "../utils/settingsHandlers"
|
||||
@@ -45,19 +46,6 @@ const ConnectionStatusIndicator = ({
|
||||
)
|
||||
}
|
||||
|
||||
const CollapsibleContent = styled.div<{ isOpen: boolean }>`
|
||||
overflow: hidden;
|
||||
transition:
|
||||
max-height 0.3s ease-in-out,
|
||||
opacity 0.3s ease-in-out,
|
||||
margin-top 0.3s ease-in-out,
|
||||
visibility 0.3s ease-in-out;
|
||||
max-height: ${({ isOpen }) => (isOpen ? "1000px" : "0")}; // Sufficiently large height
|
||||
opacity: ${({ isOpen }) => (isOpen ? 1 : 0)};
|
||||
margin-top: ${({ isOpen }) => (isOpen ? "15px" : "0")};
|
||||
visibility: ${({ isOpen }) => (isOpen ? "visible" : "hidden")};
|
||||
`
|
||||
|
||||
export const BrowserSettingsSection: React.FC<BrowserSettingsSectionProps> = ({ renderSectionHeader }) => {
|
||||
const { browserSettings } = useExtensionState()
|
||||
const [isCheckingConnection, setIsCheckingConnection] = useState(false)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { SUPPORTED_DICTATION_LANGUAGES } from "@shared/DictationSettings"
|
||||
import { McpDisplayMode } from "@shared/McpDisplayMode"
|
||||
import { OpenaiReasoningEffort } from "@shared/storage/types"
|
||||
import { VSCodeCheckbox, VSCodeDropdown, VSCodeOption, VSCodeTextField } from "@vscode/webview-ui-toolkit/react"
|
||||
@@ -7,6 +8,7 @@ import { useExtensionState } from "@/context/ExtensionStateContext"
|
||||
import Section from "../Section"
|
||||
import { updateSetting } from "../utils/settingsHandlers"
|
||||
|
||||
// import CollapsibleContent from "../CollapsibleContent"
|
||||
interface FeatureSettingsSectionProps {
|
||||
renderSectionHeader: (tabId: string) => JSX.Element | null
|
||||
}
|
||||
@@ -19,6 +21,7 @@ const FeatureSettingsSection = ({ renderSectionHeader }: FeatureSettingsSectionP
|
||||
mcpResponsesCollapsed,
|
||||
openaiReasoningEffort,
|
||||
strictPlanModeEnabled,
|
||||
dictationSettings,
|
||||
useAutoCondense,
|
||||
focusChainSettings,
|
||||
focusChainFeatureFlagEnabled,
|
||||
@@ -109,7 +112,7 @@ const FeatureSettingsSection = ({ renderSectionHeader }: FeatureSettingsSectionP
|
||||
<VSCodeOption value="high">High</VSCodeOption>
|
||||
</VSCodeDropdown>
|
||||
<p className="text-xs mt-[5px] text-[var(--vscode-descriptionForeground)]">
|
||||
Reasoning effort for the OpenAI family of models(applies to all OpenAI model providers)
|
||||
Reasoning effort for the OpenAI family of models (applies to all OpenAI model providers)
|
||||
</p>
|
||||
</div>
|
||||
<div style={{ marginTop: 10 }}>
|
||||
@@ -168,6 +171,59 @@ const FeatureSettingsSection = ({ renderSectionHeader }: FeatureSettingsSectionP
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
<div style={{ marginTop: 10 }}>
|
||||
<VSCodeCheckbox
|
||||
checked={dictationSettings?.dictationEnabled}
|
||||
onChange={(e: any) => {
|
||||
const checked = e.target.checked === true
|
||||
const updatedDictationSettings = {
|
||||
...dictationSettings,
|
||||
dictationEnabled: checked,
|
||||
}
|
||||
updateSetting("dictationSettings", updatedDictationSettings)
|
||||
}}>
|
||||
Enable Dictation
|
||||
</VSCodeCheckbox>
|
||||
<p className="text-xs text-[var(--vscode-descriptionForeground)] mt-1">
|
||||
Enables speech-to-text transcription using your Cline account. Uses the Whisper model, at $0.006
|
||||
credits per minute of audio processed. 5 minutes max per message.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* TODO: Fix and use CollapsibleContent, the animation is good but it breaks the dropdown
|
||||
<CollapsibleContent isOpen={dictationSettings?.dictationEnabled}> */}
|
||||
{dictationSettings?.dictationEnabled && (
|
||||
<div style={{ marginTop: 10, marginLeft: 20 }}>
|
||||
<label
|
||||
className="block text-sm font-medium text-[var(--vscode-foreground)] mb-1"
|
||||
htmlFor="dictation-language-dropdown">
|
||||
Dictation Language
|
||||
</label>
|
||||
<VSCodeDropdown
|
||||
className="w-full"
|
||||
currentValue={dictationSettings?.dictationLanguage || "en"}
|
||||
id="dictation-language-dropdown"
|
||||
onChange={(e: any) => {
|
||||
const newValue = e.target.value
|
||||
const updatedDictationSettings = {
|
||||
...dictationSettings,
|
||||
dictationLanguage: newValue,
|
||||
}
|
||||
updateSetting("dictationSettings", updatedDictationSettings)
|
||||
}}>
|
||||
{SUPPORTED_DICTATION_LANGUAGES.map((language) => (
|
||||
<VSCodeOption className="py-0.5" key={language.code} value={language.code}>
|
||||
{language.name}
|
||||
</VSCodeOption>
|
||||
))}
|
||||
</VSCodeDropdown>
|
||||
<p className="text-xs mt-1 text-[var(--vscode-descriptionForeground)]">
|
||||
The language you want to speak to the Dictation service in. This is separate from your preferred
|
||||
UI language.
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{/* </CollapsibleContent> */}
|
||||
<div style={{ marginTop: 10 }}>
|
||||
<VSCodeCheckbox
|
||||
checked={useAutoCondense}
|
||||
|
||||
@@ -4,6 +4,7 @@ import "../../../src/shared/webview/types"
|
||||
import { DEFAULT_AUTO_APPROVAL_SETTINGS } from "@shared/AutoApprovalSettings"
|
||||
import { findLastIndex } from "@shared/array"
|
||||
import { DEFAULT_BROWSER_SETTINGS } from "@shared/BrowserSettings"
|
||||
import { DEFAULT_DICTATION_SETTINGS, DictationSettings } from "@shared/DictationSettings"
|
||||
import { DEFAULT_PLATFORM, type ExtensionState } from "@shared/ExtensionMessage"
|
||||
import { DEFAULT_FOCUS_CHAIN_SETTINGS } from "@shared/FocusChainSettings"
|
||||
import { DEFAULT_MCP_DISPLAY_MODE } from "@shared/McpDisplayMode"
|
||||
@@ -70,6 +71,7 @@ interface ExtensionStateContextType extends ExtensionState {
|
||||
setGlobalWorkflowToggles: (toggles: Record<string, boolean>) => void
|
||||
setMcpMarketplaceCatalog: (value: McpMarketplaceCatalog) => void
|
||||
setTotalTasksSize: (value: number | null) => void
|
||||
setDictationSettings: (value: DictationSettings) => void
|
||||
|
||||
// Refresh functions
|
||||
refreshOpenRouterModels: () => void
|
||||
@@ -174,6 +176,7 @@ export const ExtensionStateContextProvider: React.FC<{
|
||||
shouldShowAnnouncement: false,
|
||||
autoApprovalSettings: DEFAULT_AUTO_APPROVAL_SETTINGS,
|
||||
browserSettings: DEFAULT_BROWSER_SETTINGS,
|
||||
dictationSettings: DEFAULT_DICTATION_SETTINGS,
|
||||
focusChainSettings: DEFAULT_FOCUS_CHAIN_SETTINGS,
|
||||
focusChainFeatureFlagEnabled: false,
|
||||
preferredLanguage: "English",
|
||||
@@ -703,6 +706,11 @@ export const ExtensionStateContextProvider: React.FC<{
|
||||
refreshOpenRouterModels,
|
||||
onRelinquishControl,
|
||||
setUserInfo: (userInfo?: UserInfo) => setState((prevState) => ({ ...prevState, userInfo })),
|
||||
setDictationSettings: (value: DictationSettings) =>
|
||||
setState((prevState) => ({
|
||||
...prevState,
|
||||
dictationSettings: value,
|
||||
})),
|
||||
}
|
||||
|
||||
return <ExtensionStateContext.Provider value={contextValue}>{children}</ExtensionStateContext.Provider>
|
||||
|
||||
@@ -62,3 +62,16 @@ export function formatSize(bytes?: number) {
|
||||
|
||||
return prettyBytes(bytes)
|
||||
}
|
||||
|
||||
export function formatSeconds(seconds?: number): string {
|
||||
if (seconds === undefined) {
|
||||
return "--:--"
|
||||
}
|
||||
|
||||
const mins = Math.floor(seconds / 60)
|
||||
const secs = Math.floor(seconds % 60)
|
||||
.toString()
|
||||
.padStart(2, "0")
|
||||
|
||||
return `${mins}:${secs}`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user