Compare commits

...
Author SHA1 Message Date
abeatrix b31a57d483 shared type 2026-02-04 09:28:13 +08:00
Bee 283aa042e8 Merge branch 'main' into bee/a-b-enrollments 2026-02-04 09:08:45 +08:00
abeatrix 62a4bf1aa3 fix import 2026-02-03 10:52:04 +08:00
abeatrix 191cfc4abd feat: A/B test enrollment tracking for feature flags
## Summary

- Add `logEnrollmentEvent()` method to `FeatureFlagsService` for tracking A/B test enrollment when users first encounter a feature
- Add `enrollmenetHistory` to global state to persist which variant (treatment/control) users are enrolled in
- Add `isPolling` flag to prevent race conditions during concurrent feature flag polling

## Test plan

- [ ] Verify `logEnrollmentEvent()` only logs enrollment once per feature flag
- [ ] Verify enrollment persists across extension restarts via `enrollmenetHistory` state
- [ ] Verify concurrent `poll()` calls don't cause race conditions
2026-02-03 10:45:59 +08:00
12 changed files with 129 additions and 33 deletions
+10 -1
View File
@@ -161,8 +161,17 @@
},
"cli": {
"name": "cline",
"version": "2.0.1",
"version": "2.0.3",
"cpu": [
"x64",
"arm64"
],
"license": "Apache-2.0",
"os": [
"darwin",
"linux",
"win32"
],
"dependencies": {
"@agentclientprotocol/sdk": "^0.13.1",
"@vscode/ripgrep": "^1.15.9",
+1 -1
View File
@@ -101,7 +101,7 @@ message Secrets {
optional string oca_refresh_token = 42;
optional string mcp_o_auth_secrets = 43;
optional string cline_api_key = 44;
optional string openai_codex_oauth_credentials = 46;
optional string openai_codex_oauth_credentials = 47;
}
// NOTE: Add new fields under API_HANDLER_SETTINGS_FIELDS or USER_SETTINGS_FIELDS
+17
View File
@@ -17,6 +17,7 @@ import {
} from "@shared/storage/state-keys"
import chokidar, { FSWatcher } from "chokidar"
import type { ExtensionContext } from "vscode"
import { FeatureFlag } from "@/shared/services/feature-flags"
import { Logger } from "@/shared/services/Logger"
import { secretStorage } from "@/shared/storage/ClineSecretStorage"
import {
@@ -919,4 +920,20 @@ export class StateManager {
}
return { ...this.workspaceStateCache }
}
public getEnrollmentHistory(flag: FeatureFlag): "treatment" | "control" | undefined {
return this.globalStateCache.enrollmenetHistory[flag] || undefined
}
public async setEnrollmentHistory(flag: FeatureFlag, value: "treatment" | "control"): Promise<boolean> {
try {
const updatedHistory = { ...this.globalStateCache.enrollmenetHistory, [flag]: value }
this.setGlobalState("enrollmenetHistory", updatedHistory)
// Immediately flush to ensure enrollment is recorded
await this.flushPendingState()
return true
} catch {
return false
}
}
}
@@ -1,8 +1,8 @@
import { ClineEndpoint } from "@/config"
import { isPostHogConfigValid, posthogConfig } from "@/shared/services/config/posthog-config"
import type { FeatureFlagsAndPayloads, IFeatureFlagsProvider } from "@/shared/services/feature-flags"
import { Logger } from "@/shared/services/Logger"
import { PostHogClientProvider } from "../telemetry/providers/posthog/PostHogClientProvider"
import type { FeatureFlagsAndPayloads, IFeatureFlagsProvider } from "./providers/IFeatureFlagsProvider"
import { PostHogFeatureFlagsProvider } from "./providers/PostHogFeatureFlagsProvider"
/**
@@ -1,9 +1,10 @@
import { clearOnboardingModelsCache, getClineOnboardingModels } from "@/core/controller/models/getClineOnboardingModels"
import { StateManager } from "@/core/storage/StateManager"
import type { OnboardingModel } from "@/shared/proto/cline/state"
import { FEATURE_FLAGS, FeatureFlag, FeatureFlagDefaultValue } from "@/shared/services/feature-flags/feature-flags"
import type { FeatureFlagPayload, FeatureFlagsAndPayloads, IFeatureFlagsProvider } from "@/shared/services/feature-flags"
import { FEATURE_FLAGS, FeatureFlag, FeatureFlagDefaultValue } from "@/shared/services/feature-flags"
import { Logger } from "@/shared/services/Logger"
import { telemetryService } from "../telemetry"
import type { FeatureFlagPayload, FeatureFlagsAndPayloads, IFeatureFlagsProvider } from "./providers/IFeatureFlagsProvider"
// Default cache time-to-live (TTL) for feature flags - an hour
const DEFAULT_CACHE_TTL = 60 * 60 * 1000
@@ -32,11 +33,20 @@ export class FeatureFlagsService {
* Tracks cache update time and user ID for cache validity
*/
private cacheInfo: CacheInfo = { updateTime: 0, userId: null }
/**
* Tracks whether a poll is currently in progress to prevent race conditions
*/
private isPolling = false
/**
* Poll all known feature flags to update their cached values
*/
public async poll(userId: string | null): Promise<void> {
// Prevent concurrent polling
if (this.isPolling) {
return
}
// Do not update cache if last update was less than an hour ago
const timesNow = Date.now()
if (timesNow - this.cacheInfo.updateTime < DEFAULT_CACHE_TTL && this.cache.size) {
@@ -46,6 +56,8 @@ export class FeatureFlagsService {
}
}
this.isPolling = true
// Only update timestamp after successfully populating cache
this.cacheInfo = { updateTime: timesNow, userId: userId || null }
@@ -63,11 +75,39 @@ export class FeatureFlagsService {
// On error, clear cache info to force refresh on next poll
this.cacheInfo = { updateTime: 0, userId: null }
throw error
} finally {
this.isPolling = false
}
getClineOnboardingModels() // Refresh onboarding models cache if relevant flag changed
}
/**
* Log an enrollment event for a feature flag experiment.
*
* Call this method when you need to enroll users in an A/B test. The enrollment
* should happen at the point where the user first encounters the feature, not at
* startup. This ensures accurate experiment tracking by only enrolling users who
* actually reach the relevant code path.
*
* @param flag The feature flag being enrolled in
* @returns True if the user was already enrolled, false if this is the first enrollment event
*/
public logEnrollmentEvent(flag: FeatureFlag): boolean {
const stateManager = StateManager.get()
const isEnrolled = stateManager.getEnrollmentHistory(flag)
// Only log if we haven't already enrolled this user for this flag
if (isEnrolled === undefined) {
const isEnabled = this.cache.get(flag) === true
const variant = isEnabled ? "treatment" : "control"
stateManager.setEnrollmentHistory(flag, variant)
telemetryService.captureRequired(`cline.experiment.${flag}`, { variant })
}
return isEnrolled !== undefined
}
private async getFeatureFlag(flagName: FeatureFlag): Promise<FeatureFlagPayload | undefined> {
try {
const payload = this.cacheInfo.flagsPayload?.featureFlagPayloads?.[flagName]
+1 -1
View File
@@ -1,10 +1,10 @@
export type { FeatureFlagsSettings, IFeatureFlagsProvider } from "@/shared/services/feature-flags"
export {
type FeatureFlagsProviderConfig,
FeatureFlagsProviderFactory,
type FeatureFlagsProviderType,
} from "./FeatureFlagsProviderFactory"
export { FeatureFlagsService } from "./FeatureFlagsService"
export type { FeatureFlagsSettings, IFeatureFlagsProvider } from "./providers/IFeatureFlagsProvider"
export { PostHogFeatureFlagsProvider } from "./providers/PostHogFeatureFlagsProvider"
import { FeatureFlagsProviderFactory } from "./FeatureFlagsProviderFactory"
@@ -3,31 +3,7 @@
* Allows switching between different feature flag providers (PostHog, etc.)
*/
/**
* Feature flags settings that control how feature flags are retrieved
*/
export interface FeatureFlagsSettings {
/** Whether feature flags are enabled */
enabled: boolean
/** Optional timeout for feature flag requests */
timeout?: number
}
type JsonType =
| string
| number
| boolean
| null
| {
[key: string]: JsonType
}
| Array<JsonType>
| JsonType[]
export type FeatureFlagPayload = string | number | boolean | { [key: string]: JsonType } | JsonType[] | null
export type FeatureFlagsAndPayloads = {
featureFlags?: Record<string, FeatureFlagPayload>
featureFlagPayloads?: Record<string, FeatureFlagPayload>
}
import { FeatureFlagsAndPayloads, FeatureFlagsSettings } from "@/shared/services/feature-flags"
/**
* Abstract interface for feature flags providers
@@ -1,8 +1,8 @@
import { PostHog } from "posthog-node"
import { getDistinctId } from "@/services/logging/distinctId"
import type { FeatureFlagsAndPayloads, FeatureFlagsSettings, IFeatureFlagsProvider } from "@/shared/services/feature-flags"
import { Logger } from "@/shared/services/Logger"
import { posthogConfig } from "../../../shared/services/config/posthog-config"
import type { FeatureFlagsAndPayloads, FeatureFlagsSettings, IFeatureFlagsProvider } from "./IFeatureFlagsProvider"
/**
* PostHog implementation of the feature flags provider interface
@@ -1,4 +1,4 @@
import type { FeatureFlagPayload } from "@/services/feature-flags/providers/IFeatureFlagsProvider"
import { FeatureFlagPayload } from "./provider"
export enum FeatureFlag {
WEBTOOLS = "webtools",
@@ -0,0 +1,2 @@
export * from "./feature-flags"
export * from "./provider"
@@ -0,0 +1,50 @@
/**
* Feature flags settings that control how feature flags are retrieved
*/
export interface FeatureFlagsSettings {
/** Whether feature flags are enabled */
enabled: boolean
/** Optional timeout for feature flag requests */
timeout?: number
}
type JsonType =
| string
| number
| boolean
| null
| {
[key: string]: JsonType
}
| Array<JsonType>
| JsonType[]
export type FeatureFlagPayload = string | number | boolean | { [key: string]: JsonType } | JsonType[] | null
export type FeatureFlagsAndPayloads = {
featureFlags?: Record<string, FeatureFlagPayload>
featureFlagPayloads?: Record<string, FeatureFlagPayload>
}
/**
* Abstract interface for feature flags providers
* Any feature flags provider must implement this interface
*/
export interface IFeatureFlagsProvider {
getAllFlagsAndPayloads(options: { flagKeys?: string[] }): Promise<FeatureFlagsAndPayloads | undefined>
/**
* Check if the provider is enabled and ready
* @returns Boolean indicating whether the provider is enabled
*/
isEnabled(): boolean
/**
* Get current feature flags settings
* @returns Current feature flags settings
*/
getSettings(): FeatureFlagsSettings
/**
* Clean up resources when the provider is disposed
*/
dispose(): Promise<void>
}
+2
View File
@@ -19,6 +19,7 @@ import { Mode, OpenaiReasoningEffort } from "@shared/storage/types"
import { TelemetrySetting } from "@shared/TelemetrySetting"
import { UserInfo } from "@shared/UserInfo"
import { LanguageModelChatSelector } from "vscode"
import { FeatureFlag } from "../services/feature-flags"
import { BlobStoreSettings } from "./ClineBlobStorage"
// ============================================================================
@@ -76,6 +77,7 @@ const GLOBAL_STATE_FIELDS = {
default: "vscodeTerminal" as "vscodeTerminal" | "backgroundExec",
},
isNewUser: { default: true as boolean },
enrollmenetHistory: { default: {} as Record<FeatureFlag, "treatment" | "control"> },
welcomeViewCompleted: { default: undefined as boolean | undefined },
mcpDisplayMode: { default: DEFAULT_MCP_DISPLAY_MODE as McpDisplayMode },
workspaceRoots: { default: undefined as WorkspaceRoot[] | undefined },