feat: Implements new taxonomy experiment for sales-and-marketing responders (#41712)

This commit is contained in:
Joco-95
2026-10-23 09:07:22 +00:00
committed by GitHub
parent 1c234b0269
commit 14e79fa897
18 changed files with 900 additions and 156 deletions
@@ -45,6 +45,7 @@ const setupSnapshotProps = {
const freeNudgeVariant = z.enum(['control', 'variant-1', 'variant-2']);
const freeNudgeTreatmentVariant = z.enum(['variant-1', 'variant-2']);
const workflowPreviewsAboveAssistantVariant = z.enum(['control', 'variant']);
const taxonomyInspirationVariant = z.enum(['control', 'variant', 'variant-bis']);
const assistantMentionKind = z.enum(['workflow', 'node', 'group']);
const assistantMentionTriggerSource = z.enum(['typed', 'button']);
/** Cap on the query text the empty-search event carries; the emitter cuts to it after redaction. */
@@ -207,6 +208,20 @@ export const INSTANCE_AI_TELEMETRY = defineTelemetryEvents({
}),
},
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
INSPIRATION_FROM_TAXONOMY_EXPOSED: {
name: 'AI Assistant inspiration from taxonomy exposed',
description:
'A sales-and-marketing respondent assigned to 129_aia_inspiration_from_taxonomy_v2 reached the n8n Assistant empty state, the view that shows the prompt suggestions, including the control variant. Fires once per visit. Does not fire on a thread, nor for the proactive starter, split empty-state, n8n Chat agent picker, or workflow-preview layouts, which replace this suggestion list. Not fired for any other experiment.',
properties: z.object({
variant: taxonomyInspirationVariant.describe(
'Variant of the 129_aia_inspiration_from_taxonomy_v2 feature flag the user saw: control, variant, or variant-bis',
),
// eslint-disable-next-line @typescript-eslint/naming-convention -- PostHog feature property
'$feature/129_aia_inspiration_from_taxonomy_v2': taxonomyInspirationVariant,
}),
},
// EOF Experiment cleanup
FREE_NUDGE_DISMISSED: {
name: 'Instance AI free nudge dismissed',
description: 'The user dismissed a visible Instance AI free-use nudge.',
@@ -1432,6 +1432,8 @@
"experiments.instanceAiPersonalizedPromptSuggestions.fallbackSuggestions.qualifyInboundLeads.description": "Research, score, and route new leads.",
"experiments.instanceAiPersonalizedPromptSuggestions.fallbackSuggestions.scheduleSocialPosts.description": "Turn weekly ideas into scheduled posts.",
"experiments.instanceAiPersonalizedPromptSuggestions.fallbackSuggestions.whatsappSupportAgent.description": "Reply to common support questions instantly.",
"experiments.instanceAiInspirationFromTaxonomy.everyone": "For everyone",
"experiments.instanceAiInspirationFromTaxonomy.heading": "For sales and marketing",
"experiments.instanceAiPersonalizedPromptSuggestions.heading": "Start from an example",
"experiments.instanceAiPersonalizedPromptSuggestions.helper": "Picked for you. Change anything.",
"experiments.instanceAiPersonalizedPromptSuggestions.seeMore": "See more",
@@ -136,9 +136,16 @@ export const OPEN_WORKFLOW_IN_ASSISTANT_EXPERIMENT = createExperiment(
'108_open_workflow_in_assistant',
);
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
export const INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT = createExperiment(
'112_aia_inspiration_from_taxonomy',
'129_aia_inspiration_from_taxonomy_v2',
{
control: 'control',
variant: 'variant',
variantBis: 'variant-bis',
},
);
// EOF Experiment cleanup
export const INSTANCE_AI_PROGRESSIVE_BUILDING_EXPERIMENT = createExperiment(
INSTANCE_AI_PROGRESSIVE_BUILDING_FLAG,
@@ -237,7 +244,9 @@ export const EXPERIMENTS_TO_TRACK = [
INLINE_AGENTS_EXPERIMENT.name,
INSTANCE_AI_FREE_NUDGE_EXPERIMENT.name,
OPEN_WORKFLOW_IN_ASSISTANT_EXPERIMENT.name,
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.name,
// EOF Experiment cleanup
MCP_JSON_NUDGE_EXPERIMENT.name,
CREDENTIAL_DESCRIPTIONS_EXPERIMENT.name,
ASSISTANT_SKILLS_EXPERIMENT.name,
@@ -8,6 +8,9 @@ import {
// Experiment cleanup (123_surface_mcp_to_claude_trial_users)
MCP_DISCOVERY_EXPERIMENT,
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT,
// EOF Experiment cleanup
LOCAL_STORAGE_EXPERIMENT_OVERRIDES,
SURFACE_ASSISTANT_ON_WORKFLOW_ERROR_EXPERIMENT, // Experiment cleanup (119_surface_assistant_on_workflow_error)
} from '@/app/constants';
@@ -578,6 +581,45 @@ describe('Posthog store', () => {
});
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
describe('taxonomy inspiration eligibility', () => {
const key = INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.name;
const participationEvent = TELEMETRY_EVENT.PLATFORM.USER_IS_PART_OF_EXPERIMENT;
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
it('does not record participation before the user is known to be sales and marketing', () => {
const posthog = usePostHog();
posthog.init({ [key]: 'variant' });
vi.advanceTimersByTime(2000);
expect(posthog.getVariant(key)).toBe('variant');
expect(useTelemetry().track).not.toHaveBeenCalledWith(
participationEvent,
expect.objectContaining({ name: key }),
);
});
it.each(['control', 'variant'] as const)(
'records participation once for an eligible %s user',
(variant) => {
const posthog = usePostHog();
posthog.init({ [key]: variant });
vi.advanceTimersByTime(100);
posthog.setTaxonomyInspirationEligible(true);
posthog.setTaxonomyInspirationEligible(true);
vi.advanceTimersByTime(2000);
expect(useTelemetry().track).toHaveBeenCalledExactlyOnceWith(participationEvent, {
name: key,
variant,
});
},
);
});
// EOF Experiment cleanup
// Experiment cleanup (119_surface_assistant_on_workflow_error)
describe('cloud-only experiment tracking', () => {
const flags = { [SURFACE_ASSISTANT_ON_WORKFLOW_ERROR_EXPERIMENT.name]: 'variant' };
@@ -10,6 +10,9 @@ import {
// Experiment cleanup (123_surface_mcp_to_claude_trial_users)
MCP_DISCOVERY_EXPERIMENT,
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT,
// EOF Experiment cleanup
EXPERIMENTS_TO_TRACK,
LOCAL_STORAGE_EXPERIMENT_OVERRIDES,
SURFACE_ASSISTANT_ON_WORKFLOW_ERROR_EXPERIMENT, // Experiment cleanup (119_surface_assistant_on_workflow_error)
@@ -40,6 +43,10 @@ export const usePostHog = defineStore('posthog', () => {
// Experiment cleanup (123_surface_mcp_to_claude_trial_users)
const mcpDiscoveryEligible = ref(false);
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
// Sales and Marketing respondents only. Stays false until the survey answer is known.
const taxonomyInspirationEligible = ref(false);
// EOF Experiment cleanup
const featureFlags: Ref<FeatureFlags | null> = ref(null);
const featureFlagPayloads: Ref<FeatureFlagPayloads | null> = ref(null);
const trackedDemoExp: Ref<FeatureFlags> = ref({});
@@ -71,6 +78,9 @@ export const usePostHog = defineStore('posthog', () => {
// Experiment cleanup (123_surface_mcp_to_claude_trial_users)
mcpDiscoveryEligible.value = false;
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
taxonomyInspirationEligible.value = false;
// EOF Experiment cleanup
featureFlagPayloads.value = null;
trackedDemoExp.value = {};
trackedExposures.value = {};
@@ -188,6 +198,14 @@ export const usePostHog = defineStore('posthog', () => {
// Experiment cleanup (123_surface_mcp_to_claude_trial_users)
if (name === MCP_DISCOVERY_EXPERIMENT.name && !mcpDiscoveryEligible.value) return;
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
if (
name === INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.name &&
!taxonomyInspirationEligible.value
) {
return;
}
// EOF Experiment cleanup
// Experiment cleanup (123_surface_mcp_to_claude_trial_users): restore const variant = featFlags[name].
const variant = name === MCP_DISCOVERY_EXPERIMENT.name ? getVariant(name) : featFlags[name];
if (!variant || trackedDemoExp.value[name] === variant) {
@@ -208,6 +226,17 @@ export const usePostHog = defineStore('posthog', () => {
if (eligible) trackExperiment(featureFlags.value ?? {}, MCP_DISCOVERY_EXPERIMENT.name);
};
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
const setTaxonomyInspirationEligible = (eligible: boolean) => {
taxonomyInspirationEligible.value = eligible;
if (eligible) {
trackExperiment(
featureFlags.value ?? {},
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.name,
);
}
};
// EOF Experiment cleanup
const trackExperiments = (featFlags: FeatureFlags) => {
EXPERIMENTS_TO_TRACK.forEach((name) => trackExperiment(featFlags, name));
@@ -360,6 +389,9 @@ export const usePostHog = defineStore('posthog', () => {
// Experiment cleanup (123_surface_mcp_to_claude_trial_users)
setMcpDiscoveryEligible,
// EOF Experiment cleanup
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
setTaxonomyInspirationEligible,
// EOF Experiment cleanup
isFeatureEnabled,
isVariantEnabled,
getVariant,
@@ -0,0 +1,87 @@
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import { fireEvent } from '@testing-library/vue';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { createComponentRenderer } from '@/__tests__/render';
import { INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS } from '../prompts';
import TaxonomyPromptSuggestions from './TaxonomyPromptSuggestions.vue';
const suggestions = INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS.map(
({ id, shortTitle, description, builderPrompt, section }) => ({
id,
shortTitle,
description,
builderPrompt,
section,
}),
);
const render = createComponentRenderer(TaxonomyPromptSuggestions, {
props: { suggestions, disabled: false },
});
const roleTitles = [
'Weekly report on my ads and website, with what changed',
'Keep my CRM updated without manual entry',
'AI chatbot that answers messages, qualifies leads and books calls',
'Schedule LinkedIn posts in my voice, after my approval',
];
describe('TaxonomyPromptSuggestions', () => {
afterEach(() => {
vi.restoreAllMocks();
});
it('renders the sales and marketing heading and one row per suggestion', () => {
const { getByText, getAllByRole, queryByText } = render();
expect(getByText('For sales and marketing')).toBeVisible();
expect(getByText('For everyone')).toBeVisible();
expect(getAllByRole('button')).toHaveLength(6);
expect(getByText('Weekly report on my ads and website, with what changed')).toBeVisible();
expect(getByText('Morning brief from Slack, Gmail and the news')).toBeVisible();
expect(getByText('Pull invoices from my email into a sheet, with due reminders')).toBeVisible();
expect(queryByText('Every Monday at 9:00')).not.toBeInTheDocument();
});
it('shuffles the sales and marketing rows and keeps For everyone in catalog order', () => {
vi.spyOn(Math, 'random').mockReturnValue(0);
const { getAllByRole } = render();
const labels = getAllByRole('button').map((button) => button.textContent);
expect(labels.slice(0, 4)).toEqual([
'Keep my CRM updated without manual entry',
'AI chatbot that answers messages, qualifies leads and books calls',
'Schedule LinkedIn posts in my voice, after my approval',
'Weekly report on my ads and website, with what changed',
]);
expect(labels.slice(4)).toEqual([
'Morning brief from Slack, Gmail and the news',
'Pull invoices from my email into a sheet, with due reminders',
]);
expect(labels.slice(0, 4).sort()).toEqual([...roleTitles].sort());
});
it('inserts the clicked suggestion prompt at its displayed position', async () => {
const { getAllByRole, emitted } = render();
const buttons = getAllByRole('button');
const index = buttons.findIndex((button) =>
button.textContent?.includes('Weekly report on my ads and website, with what changed'),
);
await fireEvent.click(buttons[index]!);
expect(emitted()['insert-suggestion']).toEqual([
[
expect.objectContaining({
suggestionId: 'v6-taxonomy-sales-and-marketing-1-weekly-ads-and-website-report',
suggestionKind: 'prompt',
suggestionTitle: 'Weekly report on my ads and website, with what changed',
position: index + 1,
prefillType: 'suggestion_catalog',
}),
],
]);
});
});
@@ -0,0 +1,263 @@
<!-- Experiment cleanup (129_aia_inspiration_from_taxonomy_v2) -->
<script lang="ts" setup>
import { computed, ref, watch } from 'vue';
import { N8nIcon, type IconName } from '@n8n/design-system';
import { useI18n, type BaseTextKey } from '@n8n/i18n';
import type { TaxonomyDisplaySuggestion } from '../metadata';
import type { InstanceAiPrefillDeclaration } from '@/features/ai/instanceAi/prefills';
const SUGGESTION_ENTER_STAGGER_MS = 45;
const SUGGESTION_ICONS: Record<string, IconName> = {
'v6-taxonomy-sales-and-marketing-1-weekly-ads-and-website-report': 'trending-up',
'v6-taxonomy-sales-and-marketing-2-keep-crm-updated': 'refresh-cw',
'v6-taxonomy-sales-and-marketing-3-qualify-leads-and-book-calls': 'message-circle',
'v6-taxonomy-sales-and-marketing-4-schedule-linkedin-posts': 'calendar',
'v6-taxonomy-sales-and-marketing-5-morning-news-brief': 'rss',
'v6-taxonomy-sales-and-marketing-6-invoices-from-email': 'file-text',
'v6-taxonomy-bis-sales-and-marketing-1-personal-follow-ups': 'mail',
'v6-taxonomy-bis-sales-and-marketing-2-form-leads-to-sheet': 'inbox',
'v6-taxonomy-bis-sales-and-marketing-3-research-ideal-customers': 'search',
'v6-taxonomy-bis-sales-and-marketing-4-draft-seo-content': 'pencil',
'v6-taxonomy-bis-sales-and-marketing-5-meeting-summaries': 'video',
'v6-taxonomy-bis-sales-and-marketing-6-messages-into-tasks': 'list-checks',
};
const SECTIONS: Array<{ id: TaxonomyDisplaySuggestion['section']; headingKey: BaseTextKey }> = [
{ id: 'role', headingKey: 'experiments.instanceAiInspirationFromTaxonomy.heading' },
{ id: 'everyone', headingKey: 'experiments.instanceAiInspirationFromTaxonomy.everyone' },
];
const props = defineProps<{
suggestions: readonly TaxonomyDisplaySuggestion[];
disabled: boolean;
}>();
type InsertSuggestionPayload = InstanceAiPrefillDeclaration & {
prompt: string;
suggestionId: string;
suggestionKind: 'prompt';
suggestionTitle: string;
position: number;
};
const emit = defineEmits<{
'preview-change': [payload: { prompt: string } | null];
'insert-suggestion': [payload: InsertSuggestionPayload];
}>();
const i18n = useI18n();
function shuffle<T>(items: readonly T[]): T[] {
const copy = [...items];
for (let index = copy.length - 1; index > 0; index -= 1) {
const swapIndex = Math.floor(Math.random() * (index + 1));
const current = copy[index];
const next = copy[swapIndex];
if (current === undefined || next === undefined) continue;
copy[index] = next;
copy[swapIndex] = current;
}
return copy;
}
const roleSuggestions = computed(() =>
props.suggestions.filter((suggestion) => suggestion.section === 'role'),
);
// Shuffled once per catalog, so a re-render does not reorder the rows.
const shuffledRoleIds = ref<string[]>([]);
watch(
() => roleSuggestions.value.map((suggestion) => suggestion.id).join('\0'),
() => {
shuffledRoleIds.value = shuffle(roleSuggestions.value).map((suggestion) => suggestion.id);
},
{ immediate: true },
);
const sections = computed(() =>
SECTIONS.map((section) => {
const inSection = props.suggestions.filter((suggestion) => suggestion.section === section.id);
if (section.id !== 'role') {
return { ...section, suggestions: inSection };
}
const byId = new Map(inSection.map((suggestion) => [suggestion.id, suggestion]));
return {
...section,
suggestions: shuffledRoleIds.value.flatMap((id) => {
const suggestion = byId.get(id);
return suggestion ? [suggestion] : [];
}),
};
}).filter((section) => section.suggestions.length > 0),
);
const displayedSuggestions = computed(() =>
sections.value.flatMap((section) => section.suggestions),
);
function iconFor(suggestionId: string): IconName {
return SUGGESTION_ICONS[suggestionId] ?? 'sparkles';
}
function getSuggestionStyle(index: number) {
return { '--suggestion-enter-delay': `${index * SUGGESTION_ENTER_STAGGER_MS}ms` };
}
function previewSuggestion(suggestion: TaxonomyDisplaySuggestion) {
if (props.disabled) return;
emit('preview-change', { prompt: suggestion.builderPrompt });
}
function clearPreview() {
emit('preview-change', null);
}
function handleSuggestionClick(suggestion: TaxonomyDisplaySuggestion) {
if (props.disabled) return;
clearPreview();
const position =
displayedSuggestions.value.findIndex((candidate) => candidate.id === suggestion.id) + 1;
emit('insert-suggestion', {
prompt: suggestion.builderPrompt,
suggestionId: suggestion.id,
suggestionKind: 'prompt',
suggestionTitle: suggestion.shortTitle,
position,
prefillType: 'suggestion_catalog',
});
}
</script>
<template>
<div :class="$style.suggestions" data-test-id="instance-ai-taxonomy-prompt-suggestions">
<section v-for="section in sections" :key="section.id" :class="$style.section">
<p :class="$style.heading">{{ i18n.baseText(section.headingKey) }}</p>
<div :class="$style.list">
<button
v-for="(suggestion, index) in section.suggestions"
:key="suggestion.id"
type="button"
:class="$style.row"
:style="getSuggestionStyle(index)"
:data-test-id="`instance-ai-taxonomy-suggestion-${suggestion.id}`"
:disabled="disabled"
@click="handleSuggestionClick(suggestion)"
@mouseenter="previewSuggestion(suggestion)"
@mouseleave="clearPreview"
@focus="previewSuggestion(suggestion)"
@blur="clearPreview"
>
<span :class="$style.iconTile">
<N8nIcon :icon="iconFor(suggestion.id)" size="small" />
</span>
<span :class="$style.title">{{ suggestion.shortTitle }}</span>
<span :class="$style.arrow">
<N8nIcon icon="arrow-right" size="small" />
</span>
</button>
</div>
</section>
</div>
</template>
<style lang="scss" module>
@use '@n8n/design-system/css/mixins/motion';
.suggestions {
display: flex;
flex-direction: column;
gap: var(--spacing--md);
width: 100%;
}
.section {
display: flex;
flex-direction: column;
gap: var(--spacing--xs);
}
.heading {
margin: 0;
color: var(--text-color--subtle);
font-size: var(--font-size--xs);
font-weight: var(--font-weight--regular);
line-height: var(--line-height--md);
}
.list {
display: flex;
flex-direction: column;
gap: var(--spacing--2xs);
}
.row {
display: flex;
align-items: center;
gap: var(--spacing--xs);
width: 100%;
min-width: 0;
padding: var(--spacing--2xs) var(--spacing--sm);
border: var(--border);
border-radius: var(--radius--lg);
background-color: var(--color--background--light-3);
color: var(--text-color);
text-align: left;
cursor: pointer;
@include motion.fade-in-up;
--animation--fade-in-up--translate: var(--spacing--3xs);
animation-delay: var(--suggestion-enter-delay, 0ms);
animation-fill-mode: backwards;
&:not(:disabled):hover,
&:not(:disabled):focus-visible {
border-color: var(--color--purple-400);
outline: none;
}
&:disabled {
color: var(--text-color--disabled);
cursor: not-allowed;
}
}
.iconTile {
display: inline-flex;
flex-shrink: 0;
align-items: center;
justify-content: center;
width: var(--spacing--xl);
height: var(--spacing--xl);
border-radius: var(--radius--sm);
// Sampled from the reference tile. No purple token is this dark and muted.
background-color: #2e2b4e;
color: var(--color--purple-300);
}
.title {
flex: 1;
min-width: 0;
overflow: hidden;
color: var(--text-color);
font-size: var(--font-size--sm);
font-weight: var(--font-weight--medium);
line-height: var(--line-height--md);
text-overflow: ellipsis;
white-space: nowrap;
}
.arrow {
display: inline-flex;
flex-shrink: 0;
align-items: center;
justify-content: center;
width: var(--spacing--lg);
height: var(--spacing--lg);
border-radius: var(--radius--sm);
background-color: var(--color--foreground);
color: var(--color--text);
}
</style>
@@ -1,4 +1,9 @@
export { INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION } from './prompts';
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
export {
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS,
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS_VERSION,
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION,
} from './prompts';
export {
isPersonalizedPromptSuggestionResolution,
isTaxonomyPromptSuggestionResolution,
@@ -6,4 +11,5 @@ export {
resolveTaxonomySegment,
type TaxonomyPromptSuggestionResolution,
} from './metadata';
export { default as TaxonomyPromptSuggestions } from './components/TaxonomyPromptSuggestions.vue';
export { useInstanceAiInspirationFromTaxonomyExperiment } from './useInstanceAiInspirationFromTaxonomyExperiment';
@@ -1,12 +1,16 @@
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import { describe, expect, it } from 'vitest';
import { resolveTaxonomyPromptSuggestions } from './metadata';
import { INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS } from './prompts';
const salesAndMarketingSuggestionIds = [
'v5-taxonomy-sales-and-marketing-1-link-attio-contacts-to-their-deals',
'v5-taxonomy-sales-and-marketing-2-route-nps-feedback-to-follow-up',
'v5-taxonomy-sales-and-marketing-3-enrich-new-hubspot-companies',
'v5-taxonomy-sales-and-marketing-4-ai-coaching-feedback-on-sales-calls',
'v6-taxonomy-sales-and-marketing-1-weekly-ads-and-website-report',
'v6-taxonomy-sales-and-marketing-2-keep-crm-updated',
'v6-taxonomy-sales-and-marketing-3-qualify-leads-and-book-calls',
'v6-taxonomy-sales-and-marketing-4-schedule-linkedin-posts',
'v6-taxonomy-sales-and-marketing-5-morning-news-brief',
'v6-taxonomy-sales-and-marketing-6-invoices-from-email',
];
describe('instance AI inspiration from taxonomy metadata', () => {
@@ -21,16 +25,6 @@ describe('instance AI inspiration from taxonomy metadata', () => {
segmentKey: 'sales-and-marketing',
suggestionIds: salesAndMarketingSuggestionIds,
},
{
answer: 'IT',
segmentKey: 'it',
suggestionIds: [
'v5-taxonomy-it-1-detect-and-escalate-1password-vault-exports',
'v5-taxonomy-it-2-escalate-entra-id-high-risk-users',
'v5-taxonomy-it-3-turn-emails-into-notion-tasks',
'v5-taxonomy-it-4-alert-on-unexplained-payment-failures',
],
},
])(
'returns the $segmentKey catalog for the $answer role',
({ answer, segmentKey, suggestionIds }) => {
@@ -47,7 +41,7 @@ describe('instance AI inspiration from taxonomy metadata', () => {
expect(resolution.suggestions.map((suggestion) => suggestion.id)).toEqual(suggestionIds);
expect(resolution.showSeeMore).toBe(false);
expect(resolution.telemetryPayload).toEqual({
suggestion_catalog_version: 'v5-taxonomy',
suggestion_catalog_version: 'v6-taxonomy',
suggestion_format: 'list',
suggestion_source: 'taxonomy',
segment_key: segmentKey,
@@ -58,22 +52,56 @@ describe('instance AI inspiration from taxonomy metadata', () => {
it.each([
{ name: 'the role is missing', metadata: null },
{ name: 'the role is IT', metadata: { what_team_are_you_on: 'IT' } },
{ name: 'the role is not mapped', metadata: { what_team_are_you_on: 'Engineering' } },
])('falls back to control when $name', ({ metadata }) => {
])('excludes the user when $name', ({ metadata }) => {
expect(
resolveTaxonomyPromptSuggestions({
metadata,
metadataLoadState: 'loaded',
}),
).toEqual({ source: 'control' });
).toEqual({ source: 'excluded' });
});
it('falls back to control when metadata is not loaded', () => {
it('excludes the user when metadata is not loaded', () => {
expect(
resolveTaxonomyPromptSuggestions({
metadata: { what_team_are_you_on: 'Sales' },
metadataLoadState: 'not_cloud',
}),
).toEqual({ source: 'control' });
).toEqual({ source: 'excluded' });
});
it('returns the variant-bis catalog when that catalog is requested', () => {
const resolution = resolveTaxonomyPromptSuggestions({
metadata: { what_team_are_you_on: 'Sales' },
metadataLoadState: 'loaded',
catalog: INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS,
catalogVersion: 'v6-taxonomy-bis',
});
expect(resolution.source).toBe('taxonomy');
if (resolution.source !== 'taxonomy') return;
expect(resolution.suggestions.map((suggestion) => suggestion.shortTitle)).toEqual([
"Send personal follow-ups to leads who haven't replied",
'Save form leads to a Google Sheet, email them and alert me',
'Find and research leads that fit my ideal customer',
'Draft SEO blog posts and newsletters for my review',
'Meeting summaries and action items, sent to me',
'Turn requests in emails and messages into tasks',
]);
expect(resolution.telemetryPayload.suggestion_catalog_version).toBe('v6-taxonomy-bis');
});
it('reports the rows format for the variant layout', () => {
const resolution = resolveTaxonomyPromptSuggestions({
metadata: { what_team_are_you_on: 'Sales' },
metadataLoadState: 'loaded',
suggestionFormat: 'rows',
});
expect(resolution.source).toBe('taxonomy');
if (resolution.source !== 'taxonomy') return;
expect(resolution.telemetryPayload.suggestion_format).toBe('rows');
});
});
@@ -1,3 +1,4 @@
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import type { CloudPersonalizationMetadata } from '@/experiments/instanceAiPersonalizedPromptSuggestions/metadata';
import type {
PersonalizedPromptDisplaySuggestion,
@@ -8,28 +9,38 @@ import type {
import {
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS,
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION,
type INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS_VERSION,
TAXONOMY_SEE_MORE_ENABLED,
type TaxonomyPromptSuggestion,
type TaxonomyRole,
type TaxonomySuggestionSection,
} from './prompts';
const TEAM_METADATA_KEY = 'what_team_are_you_on';
const SUGGESTIONS_PER_BUCKET = 4;
const ROLE_SECTION_SIZE = 4;
const EVERYONE_SECTION_SIZE = 2;
export type TaxonomyDisplaySuggestion = PersonalizedPromptDisplaySuggestion & {
section: TaxonomySuggestionSection;
};
const TAXONOMY_ROLE_ANSWERS = {
Sales: 'sales-and-marketing',
Marketing: 'sales-and-marketing',
IT: 'it',
} as const satisfies Record<string, TaxonomyRole>;
export type TaxonomySuggestionFormat = 'list' | 'rows';
export type TaxonomyPromptSegment =
| { source: 'taxonomy'; taxonomyRole: TaxonomyRole; segmentKey: TaxonomyRole }
| { source: 'control' };
| { source: 'excluded' };
export type TaxonomyPromptTelemetryPayload = {
suggestion_catalog_version: typeof INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION;
suggestion_format: 'list';
suggestion_catalog_version:
| typeof INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION
| typeof INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS_VERSION;
suggestion_format: TaxonomySuggestionFormat;
suggestion_source: 'taxonomy';
segment_key: TaxonomyRole;
metadata_load_state: PersonalizedPromptMetadataLoadState;
@@ -38,11 +49,11 @@ export type TaxonomyPromptTelemetryPayload = {
export type TaxonomyPromptSuggestionResolution =
| {
source: 'taxonomy';
suggestions: PersonalizedPromptDisplaySuggestion[];
suggestions: TaxonomyDisplaySuggestion[];
showSeeMore: boolean;
telemetryPayload: TaxonomyPromptTelemetryPayload;
}
| { source: 'control' };
| { source: 'excluded' };
export function isTaxonomyPromptSuggestionResolution(
resolution: PersonalizedPromptSuggestionResolution | TaxonomyPromptSuggestionResolution | null,
@@ -61,12 +72,12 @@ export function resolveTaxonomySegment(
): TaxonomyPromptSegment {
const roleAnswer = metadata?.[TEAM_METADATA_KEY];
if (typeof roleAnswer !== 'string') {
return { source: 'control' };
return { source: 'excluded' };
}
const taxonomyRole = TAXONOMY_ROLE_ANSWERS[roleAnswer as keyof typeof TAXONOMY_ROLE_ANSWERS];
if (!taxonomyRole) {
return { source: 'control' };
return { source: 'excluded' };
}
return { source: 'taxonomy', taxonomyRole, segmentKey: taxonomyRole };
@@ -75,43 +86,58 @@ export function resolveTaxonomySegment(
function getBucketSuggestions(
taxonomyRole: TaxonomyRole,
catalog: readonly TaxonomyPromptSuggestion[],
): PersonalizedPromptDisplaySuggestion[] | null {
): TaxonomyDisplaySuggestion[] | null {
const suggestions = catalog
.filter((suggestion) => suggestion.taxonomyRole === taxonomyRole)
.sort((a, b) => a.order - b.order)
.map(({ id, shortTitle, description, builderPrompt }) => ({
.map(({ id, shortTitle, description, builderPrompt, section }) => ({
id,
shortTitle,
description,
builderPrompt,
section,
}));
const roleCount = suggestions.filter((suggestion) => suggestion.section === 'role').length;
const everyoneCount = suggestions.filter(
(suggestion) => suggestion.section === 'everyone',
).length;
return suggestions.length === SUGGESTIONS_PER_BUCKET ? suggestions : null;
if (roleCount !== ROLE_SECTION_SIZE || everyoneCount !== EVERYONE_SECTION_SIZE) {
return null;
}
return suggestions;
}
const CONTROL_RESOLUTION = { source: 'control' } as const;
const EXCLUDED_RESOLUTION = { source: 'excluded' } as const;
export function resolveTaxonomyPromptSuggestions({
metadata,
metadataLoadState,
suggestionFormat = 'list',
catalog = INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS,
catalogVersion = INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION,
}: {
metadata: CloudPersonalizationMetadata;
metadataLoadState: PersonalizedPromptMetadataLoadState;
suggestionFormat?: TaxonomySuggestionFormat;
catalog?: readonly TaxonomyPromptSuggestion[];
catalogVersion?:
| typeof INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION
| typeof INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS_VERSION;
}): TaxonomyPromptSuggestionResolution {
if (metadataLoadState !== 'loaded') {
return CONTROL_RESOLUTION;
return EXCLUDED_RESOLUTION;
}
const segment = resolveTaxonomySegment(metadata);
if (segment.source === 'control') {
return CONTROL_RESOLUTION;
if (segment.source === 'excluded') {
return EXCLUDED_RESOLUTION;
}
const suggestions = getBucketSuggestions(segment.taxonomyRole, catalog);
if (!suggestions) {
return CONTROL_RESOLUTION;
return EXCLUDED_RESOLUTION;
}
return {
@@ -119,8 +145,8 @@ export function resolveTaxonomyPromptSuggestions({
suggestions,
showSeeMore: TAXONOMY_SEE_MORE_ENABLED,
telemetryPayload: {
suggestion_catalog_version: INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION,
suggestion_format: 'list',
suggestion_catalog_version: catalogVersion,
suggestion_format: suggestionFormat,
suggestion_source: 'taxonomy',
segment_key: segment.segmentKey,
metadata_load_state: metadataLoadState,
@@ -1,93 +1,143 @@
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import type { PersonalizedPromptDisplaySuggestion } from '@/experiments/instanceAiPersonalizedPromptSuggestions/types';
export type TaxonomyRole = 'sales-and-marketing' | 'it';
export type TaxonomyRole = 'sales-and-marketing';
export type TaxonomySuggestionSection = 'role' | 'everyone';
export type TaxonomyPromptSuggestion = PersonalizedPromptDisplaySuggestion & {
taxonomyRole: TaxonomyRole;
section: TaxonomySuggestionSection;
order: number;
};
export const INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION = 'v5-taxonomy';
export const INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION = 'v6-taxonomy';
export const INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS_VERSION = 'v6-taxonomy-bis';
export const TAXONOMY_SEE_MORE_ENABLED = false;
export const INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS = [
{
id: 'v5-taxonomy-it-1-detect-and-escalate-1password-vault-exports',
taxonomyRole: 'it',
id: 'v6-taxonomy-sales-and-marketing-1-weekly-ads-and-website-report',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 1,
shortTitle: 'Detect and escalate 1Password vault exports',
description:
'Monitor 1Password audit events for vault exports and escalates each detection to TheHive as an alert, with a notification to your Slack security channel.',
shortTitle: 'Weekly report on my ads and website, with what changed',
description: 'A Monday summary of spend, leads, and what moved versus last week.',
builderPrompt:
'Fetch recent 1Password audit events and keep only vault exports. For each export, collect the user, vault name, timestamp, and IP if available. Create a high-severity TheHive alert titled "1Password vault export detected", with the details in the description. Post a summary on Slack with a link to the TheHive alert',
"Build a workflow that sends me a weekly marketing performance report. Every Monday at 9:00, pull last week's results from my ad and analytics accounts (for example Google Ads, Meta Ads, Google Search Console or Google Analytics) and compare them with the previous week. Use AI to write a short summary covering spend, leads, cost per lead, and what went up or down. Email it to me with Gmail and also save the numbers to a Google Sheet. Before building, ask me which platforms to include.",
},
{
id: 'v5-taxonomy-it-2-escalate-entra-id-high-risk-users',
taxonomyRole: 'it',
id: 'v6-taxonomy-sales-and-marketing-2-keep-crm-updated',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 2,
shortTitle: 'Escalate Entra ID high-risk users',
description:
'Check Microsoft Entra ID for high-risk user detections, aggregate the risk details per user, open an alert in TheHive, and notify to security channel on Slack.',
shortTitle: 'Keep my CRM updated without manual entry',
description: 'Creates or updates the matching CRM record when something changes elsewhere.',
builderPrompt:
'Every 15 minutes, query Microsoft Graph for Entra ID risk detections and keep only high-risk users. Group detections per user and aggregate risk level, risk state, detection types, sign-in location/IP, and timestamps. Create a high-severity TheHive alert titled "Entra ID high-risk user detected: <user>", with the details in the description. Post a message on Slack with a link to the TheHive alert.',
"Build a workflow that keeps my CRM up to date automatically. When something changes in my other tools, such as a new contact in a spreadsheet, a new email from a client, a booked meeting or a paid invoice, find the matching contact or deal in my CRM, create it if it doesn't exist, and update the right fields without creating duplicates. Add a short note to the record describing what happened. Notify me on Slack or by email if a record can't be matched. Before building, ask me which CRM I use and which tools it should sync with.",
},
{
id: 'v5-taxonomy-it-3-turn-emails-into-notion-tasks',
taxonomyRole: 'it',
id: 'v6-taxonomy-sales-and-marketing-3-qualify-leads-and-book-calls',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 3,
shortTitle: 'Turn emails into Notion tasks',
description: 'Extract action items from labeled emails automatically.',
shortTitle: 'AI chatbot that answers messages, qualifies leads and books calls',
description: 'Answers messages, asks qualifying questions, and saves the lead.',
builderPrompt:
'On a schedule, check Gmail for messages with a specific label. Extract the task and its key details from each, then create a matching task card in a Notion database.',
'Build an AI chatbot that answers people who message me on WhatsApp or my website chat and turns them into qualified leads. It should answer basic questions about my business using information I provide, ask a few qualifying questions (what they need, budget and timeline), collect their name and email or phone, and offer a link to book a call. Save each lead to a Google Sheet or my CRM and notify me on Slack, Telegram or by email. Before building, ask me which channel to use, what my business offers and which questions qualify a lead.',
},
{
id: 'v5-taxonomy-it-4-alert-on-unexplained-payment-failures',
taxonomyRole: 'it',
id: 'v6-taxonomy-sales-and-marketing-4-schedule-linkedin-posts',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 4,
shortTitle: 'Alert on unexplained payment failures',
description:
'Watch recurring payment transactions for failures with unknown causes and send an alert with the transaction for investigation.',
shortTitle: 'Schedule LinkedIn posts in my voice, after my approval',
description: 'Drafts a weekday post and publishes it only after you approve.',
builderPrompt:
'Every hour, fetch recent failed recurring payments from my payment system. Keep only failures whose reason is unknown, missing, or uncategorized, and exclude causes such as insufficient funds or expired cards. For each one, collect the transaction ID, customer, amount, currency, timestamp, and raw error data. Alert my team in Slack.',
'Build a workflow that drafts and schedules social media posts for me. Every weekday morning, take the next topic, article or idea from my content list (for example a Google Sheet, Notion or Airtable) and use AI to write a post in my brand voice for LinkedIn. Send it to me on Slack or Telegram for approval. When I approve it, publish or schedule it and mark the item as posted in my content list. Before building, ask me which content source, social platforms and brand voice to use.',
},
{
id: 'v5-taxonomy-sales-and-marketing-1-link-attio-contacts-to-their-deals',
id: 'v6-taxonomy-sales-and-marketing-5-morning-news-brief',
taxonomyRole: 'sales-and-marketing',
order: 1,
shortTitle: 'Link Attio contacts to their deals',
description: 'Keep CRM records connected without manual data entry.',
section: 'everyone',
order: 5,
shortTitle: 'Morning brief from Slack, Gmail and the news',
description: 'A daily 8:00 summary of the sources and topics you follow.',
builderPrompt:
"On a schedule, go through my Attio contacts. Look up each contact's company and deals, then associate the contact with the right deals so my CRM records stay connected.",
'Build a workflow that sends me a daily brief every morning at 8:00. Collect the latest items from the sources I choose (for example news sites or RSS feeds, newsletters in my inbox, industry blogs, or Slack channels). Use AI to pick the most important items, summarize each in one or two sentences with a link, and group them by topic. Send the brief to me by email, Slack or Telegram. Before building, ask me which topics and sources to follow and where to send the brief.',
},
{
id: 'v5-taxonomy-sales-and-marketing-2-route-nps-feedback-to-follow-up',
id: 'v6-taxonomy-sales-and-marketing-6-invoices-from-email',
taxonomyRole: 'sales-and-marketing',
order: 2,
shortTitle: 'Route NPS feedback to the right follow-up',
description:
'Sorts incoming NPS responses: promoters get tagged in Mailchimp for follow-up campaigns, detractors trigger an Outlook alert.',
section: 'everyone',
order: 6,
shortTitle: 'Pull invoices from my email into a sheet, with due reminders',
description: 'Extracts invoice details from email and reminds you before they are due.',
builderPrompt:
'When a new NPS response arrives, evaluate the score. For 9-10 (promoter), find the customer in Mailchimp by email, add them to my main audience, and add a "happy customer" tag for referral and review campaigns. For 6 or below (detractor), email customer success through Outlook with the name, email, score, and any feedback. Do nothing for 7-8.',
'Build a workflow that collects invoices and receipts from my email automatically. When a new email arrives with an invoice or receipt attached, use AI to extract the vendor, invoice number, date, amount, currency and due date. Save the file to a Google Drive folder named by month and add a row with the details to a Google Sheet. Send me a reminder by email or Slack when an invoice is due within 3 days. Before building, ask me which email account to watch, where to save files and which spreadsheet to use.',
},
] as const satisfies readonly TaxonomyPromptSuggestion[];
export const INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS = [
{
id: 'v5-taxonomy-sales-and-marketing-3-enrich-new-hubspot-companies',
id: 'v6-taxonomy-bis-sales-and-marketing-1-personal-follow-ups',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 1,
shortTitle: "Send personal follow-ups to leads who haven't replied",
description: 'Writes a short follow-up for quiet leads and stops when they reply.',
builderPrompt:
"Build a workflow that sends personalized follow-up emails to my leads. Every weekday, check my lead list (a Google Sheet or my CRM) for leads I haven't contacted yet or who haven't replied in 3 days. For each one, use AI to write a short, personal email based on what I know about them (company, role, earlier messages), then send it from Gmail or save it as a draft for me to review. Update the lead's status and last-contact date in my list, stop when someone replies, and send at most 3 follow-ups per lead. Before building, ask me where my leads are stored and whether emails should be sent automatically or saved as drafts.",
},
{
id: 'v6-taxonomy-bis-sales-and-marketing-2-form-leads-to-sheet',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 2,
shortTitle: 'Save form leads to a Google Sheet, email them and alert me',
description: 'Stores a new form lead, confirms by email, and alerts you.',
builderPrompt:
"Build a workflow that handles new leads from my website form. Trigger it with a webhook or my form tool's trigger. Clean up the fields (name, email, phone, company, message) and add the lead as a new row in a Google Sheet, skipping duplicates. Send the lead a short, friendly confirmation email from Gmail, and send me an instant alert on Slack, Telegram or by email with the lead's details. Before building, ask me which form tool I use and where to send alerts.",
},
{
id: 'v6-taxonomy-bis-sales-and-marketing-3-research-ideal-customers',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 3,
shortTitle: 'Enrich new HubSpot companies',
description:
'Automatically researches every new company added to HubSpot using AI web search and fills in its profile with business model, category, and location data.',
shortTitle: 'Find and research leads that fit my ideal customer',
description: 'Finds matching businesses and writes why they might need your product.',
builderPrompt:
"Build an automation that enriches newly created HubSpot company records with AI research. Run whenever a new company is created in HubSpot. For each new company, take its name and website domain and use an AI model with web search capability to research the business. Structured data should be returned: the company's business model (e.g. B2B SaaS, e-commerce, agency, marketplace), its industry category, a one sentence description of what it does, its headquarters location, and approximate company size if discoverable. Update the corresponding HubSpot company record's properties with this data. If the research fails or returns low-confidence results, tag the company for manual review.",
'Build a workflow that finds new leads and researches them for me. Search a source I choose (for example Google Maps for local businesses, Apollo, or a list of company websites) for businesses that match my target customer. For each one, collect the name, website, location and contact details, then use AI to read their website and write a short summary of what they do and why they might need my product. Save everything to a Google Sheet, skipping leads I already have. Before building, ask me who my target customer is and which source to use.',
},
{
id: 'v5-taxonomy-sales-and-marketing-4-ai-coaching-feedback-on-sales-calls',
id: 'v6-taxonomy-bis-sales-and-marketing-4-draft-seo-content',
taxonomyRole: 'sales-and-marketing',
section: 'role',
order: 4,
shortTitle: 'AI coaching feedback on sales calls',
description:
'Generate structured coaching feedback and performance scores from sales calls, then posts the results to Slack.',
shortTitle: 'Draft SEO blog posts and newsletters for my review',
description: 'Writes a blog post and a newsletter draft from the next topic.',
builderPrompt:
'Build a workflow that writes content for me. Each week, take the next topic or keyword from my content list (for example a Google Sheet or Notion). Use AI to research the topic and write an SEO-friendly blog post with a title, meta description, headings and a call to action, plus a short newsletter version, all in my brand voice. Save both as drafts in Google Docs, update the status in my content list, and send me the links for review. Before building, ask me where my topics live, who my audience is and how to describe my brand voice.',
},
{
id: 'v6-taxonomy-bis-sales-and-marketing-5-meeting-summaries',
taxonomyRole: 'sales-and-marketing',
section: 'everyone',
order: 5,
shortTitle: 'Meeting summaries and action items, sent to me',
description: 'Summarizes a finished meeting and lists the action items.',
builderPrompt:
'Build a workflow that summarizes my meetings automatically. When a meeting ends and its recording or transcript is available (for example from Google Meet, Zoom, Fireflies or Fathom), use AI to write a short summary with the key decisions, action items with owners, and any follow-up dates. Send the summary to me by email or Slack and save it to a Google Doc or Notion page. Before building, ask me which meeting tool I use and where summaries should go.',
},
{
id: 'v6-taxonomy-bis-sales-and-marketing-6-messages-into-tasks',
taxonomyRole: 'sales-and-marketing',
section: 'everyone',
order: 6,
shortTitle: 'Turn requests in emails and messages into tasks',
description: 'Creates a task from a request and sends a daily due summary.',
builderPrompt:
"When a new call transcript is available, send it to an AI sales coach. Score the call 1-10 and per criterion: discovery questions, objection handling, talk-to-listen ratio, next-step commitment, and rapport. Return three strengths, three improvement areas with quotes from the call, and a suggested follow-up. Post a readable summary to Slack with the rep's name, call date, prospect/company, overall score, and coaching feedback.",
'Build a workflow that creates tasks for me automatically. When I get an email, Slack message or form submission that contains a request or action item, use AI to decide whether it needs a task. If it does, create the task in my task or project tool with a clear title, a short description, a due date if one is mentioned, and a link back to the original message. Send me a daily summary of tasks that are due or overdue. Before building, ask me which apps to watch and which task tool I use (for example Asana, Trello, ClickUp, Notion or Todoist).',
},
] as const satisfies readonly TaxonomyPromptSuggestion[];
@@ -1,3 +1,4 @@
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import { beforeEach, describe, expect, it, vi } from 'vitest';
import {
@@ -24,21 +25,29 @@ describe('useInstanceAiInspirationFromTaxonomyExperiment', () => {
{
variant: INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.variant,
isTreatmentVariant: true,
isRowsVariant: true,
},
{
variant: INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.variantBis,
isTreatmentVariant: false,
isRowsVariant: true,
},
{
variant: INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.control,
isTreatmentVariant: false,
isRowsVariant: false,
},
{ variant: undefined, isTreatmentVariant: false },
{ variant: undefined, isTreatmentVariant: false, isRowsVariant: false },
])(
'returns isTreatmentVariant $isTreatmentVariant when PostHog variant is $variant',
({ variant, isTreatmentVariant }) => {
'returns isRowsVariant $isRowsVariant when PostHog variant is $variant',
({ variant, isTreatmentVariant, isRowsVariant }) => {
getVariant.mockReturnValue(variant);
const experiment = useInstanceAiInspirationFromTaxonomyExperiment();
expect(experiment.currentVariant.value).toBe(variant);
expect(experiment.isTreatmentVariant.value).toBe(isTreatmentVariant);
expect(experiment.isRowsVariant.value).toBe(isRowsVariant);
expect(getVariant).toHaveBeenCalledWith(
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.name,
);
@@ -1,3 +1,4 @@
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import { computed } from 'vue';
import { INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT } from '@/app/constants/experiments';
@@ -12,9 +13,15 @@ export function useInstanceAiInspirationFromTaxonomyExperiment() {
const isTreatmentVariant = computed(
() => currentVariant.value === INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.variant,
);
const isRowsVariant = computed(
() =>
isTreatmentVariant.value ||
currentVariant.value === INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.variantBis,
);
return {
currentVariant,
isTreatmentVariant,
isRowsVariant,
};
}
@@ -24,8 +24,13 @@ import { countAttachedNodes } from './utils/buildNodesAttachment';
import { useDocumentTitle } from '@/app/composables/useDocumentTitle';
import { usePageRedirectionHelper } from '@/app/composables/usePageRedirectionHelper';
import { getExperimentTelemetryPayload } from '@/experiments/utils';
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import { usePostHog } from '@/app/stores/posthog.store';
// EOF Experiment cleanup
import {
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT,
// EOF Experiment cleanup
INSTANCE_AI_PERSONALIZED_PROMPT_SUGGESTIONS_EXPERIMENT,
WORKFLOW_PREVIEWS_ABOVE_ASSISTANT_EXPERIMENT,
} from '@/app/constants/experiments';
@@ -69,15 +74,20 @@ import {
type PersonalizedPromptMetadataLoadState,
type PersonalizedPromptSuggestionResolution,
} from '@/experiments/instanceAiPersonalizedPromptSuggestions';
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
import {
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS,
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS_VERSION,
INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION,
isPersonalizedPromptSuggestionResolution,
isTaxonomyPromptSuggestionResolution,
resolveTaxonomyPromptSuggestions,
resolveTaxonomySegment,
TaxonomyPromptSuggestions,
useInstanceAiInspirationFromTaxonomyExperiment,
type TaxonomyPromptSuggestionResolution,
} from '@/experiments/instanceAiInspirationFromTaxonomy';
// EOF Experiment cleanup
import {
WorkflowPreviewSuggestions,
WorkflowPreviewCanvas,
@@ -135,9 +145,6 @@ const INSTANCE_AI_SPLIT_FIXED_ROWS = 5;
const PERSONALIZED_PROMPT_METADATA_TIMEOUT_MS = 2000;
const INSTANCE_AI_PERSONALIZED_PROMPT_SUGGESTIONS_EXPOSURE_EVENT =
'Instance AI personalized prompt suggestions exposed';
const INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPOSURE_EVENT =
'Instance AI inspiration from taxonomy exposed';
const store = useInstanceAiStore();
const appSettingsStore = useSettingsStore();
const cloudPlanStore = useCloudPlanStore();
@@ -223,10 +230,20 @@ const {
isTreatmentVariant: isPersonalizedPromptSuggestionsTreatmentVariant,
suggestionFormat: personalizedPromptSuggestionsFormat,
} = useInstanceAiPersonalizedPromptSuggestionsExperiment();
const {
currentVariant: inspirationFromTaxonomyVariant,
isTreatmentVariant: isInspirationFromTaxonomyTreatmentVariant,
} = useInstanceAiInspirationFromTaxonomyExperiment();
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
const { currentVariant: inspirationFromTaxonomyVariant, isRowsVariant: isTaxonomyRowsVariant } =
useInstanceAiInspirationFromTaxonomyExperiment();
const posthogStore = usePostHog();
const isTaxonomyExperimentAssigned = computed(
() =>
inspirationFromTaxonomyVariant.value ===
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.control ||
inspirationFromTaxonomyVariant.value ===
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.variant ||
inspirationFromTaxonomyVariant.value ===
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.variantBis,
);
// EOF Experiment cleanup
const showProactiveStarter = computed(
() => isProactiveAgentExperimentEnabled.value && !forcesDefaultEmptyLayout.value,
);
@@ -247,12 +264,35 @@ const shouldTrackPersonalizedPromptSuggestionsExposure = computed(
settingsStore.isWorkflowBuilderAvailable,
);
const personalizedPromptSuggestionResolution = ref<
PersonalizedPromptSuggestionResolution | TaxonomyPromptSuggestionResolution | null
| PersonalizedPromptSuggestionResolution
| TaxonomyPromptSuggestionResolution // Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
| null
>(null);
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
const shouldShowTaxonomySuggestions = computed(() =>
isTaxonomyPromptSuggestionResolution(personalizedPromptSuggestionResolution.value),
);
const isTaxonomySegmentResolved = computed(
watch(
[
isTaxonomyExperimentAssigned,
() => appSettingsStore.isCloudDeployment,
() => cloudPlanStore.state.initialized,
() => cloudPlanStore.currentUserCloudInfo,
],
() => {
if (!isTaxonomyExperimentAssigned.value) return;
if (!appSettingsStore.isCloudDeployment || !cloudPlanStore.state.initialized) return;
const segment = resolveTaxonomySegment(
cloudPlanStore.currentUserCloudInfo?.information ?? null,
);
if (segment.source !== 'taxonomy') return;
posthogStore.setTaxonomyInspirationEligible(true);
},
{ immediate: true },
);
const isSalesAndMarketingSegment = computed(
() =>
appSettingsStore.isCloudDeployment &&
cloudPlanStore.state.initialized &&
@@ -260,29 +300,27 @@ const isTaxonomySegmentResolved = computed(
'taxonomy',
);
const shouldTrackInspirationFromTaxonomyExposure = computed(() => {
if (showProactiveStarter.value || isSplitLayoutActive.value || forcesDefaultEmptyLayout.value) {
if (!isTaxonomyExperimentAssigned.value) {
return false;
}
// Experiment cleanup (124_workflow_previews_above_assistant)
if (showWorkflowPreviewsAboveAssistant.value) {
return false;
}
// EOF Experiment cleanup
if (!settingsStore.isWorkflowBuilderAvailable) {
if (!isSalesAndMarketingSegment.value) {
return false;
}
// These layouts replace the suggestion list on the Assistant empty state.
if (
inspirationFromTaxonomyVariant.value ===
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.control
showProactiveStarter.value ||
isSplitLayoutActive.value ||
forcesDefaultEmptyLayout.value ||
showWorkflowPreviewsAboveAssistant.value
) {
return isTaxonomySegmentResolved.value;
return false;
}
return shouldShowTaxonomySuggestions.value;
return true;
});
// EOF Experiment cleanup
const activeWorkflowPreviewFile = ref<string | null>(null);
const activeWorkflowPreview = computed(() => {
if (!activeWorkflowPreviewFile.value) return null;
@@ -291,7 +329,7 @@ const activeWorkflowPreview = computed(() => {
const personalizedPromptProfileOverride = usePersonalizedPromptProfileOverride();
let personalizedPromptMetadataTimeout: ReturnType<typeof setTimeout> | null = null;
let hasTrackedPersonalizedPromptSuggestionsExposure = false;
let hasTrackedInspirationFromTaxonomyExposure = false;
let hasTrackedInspirationFromTaxonomyExposure = false; // Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
// Experiment cleanup (124_workflow_previews_above_assistant)
let hasTrackedWorkflowPreviewsEmptyStateView = false;
const shouldTrackWorkflowPreviewsEmptyStateView = computed(
@@ -304,7 +342,7 @@ const shouldTrackWorkflowPreviewsEmptyStateView = computed(
// EOF Experiment cleanup
const isAnyPersonalizedSuggestionsTreatmentActive = computed(
() =>
isInspirationFromTaxonomyTreatmentVariant.value ||
isTaxonomyRowsVariant.value || // Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
isPersonalizedPromptSuggestionsTreatmentVariant.value,
);
@@ -322,10 +360,20 @@ function clearPersonalizedPromptMetadataTimeout() {
}
function setPersonalizedPromptResolution(metadataLoadState: PersonalizedPromptMetadataLoadState) {
if (isInspirationFromTaxonomyTreatmentVariant.value) {
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
// Control keeps the default empty state. Only the variant renders the rows.
if (isTaxonomyRowsVariant.value) {
const isVariantBis =
inspirationFromTaxonomyVariant.value ===
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT.variantBis;
const taxonomyResolution = resolveTaxonomyPromptSuggestions({
metadata: cloudPlanStore.currentUserCloudInfo?.information ?? null,
metadataLoadState,
suggestionFormat: 'rows',
catalog: isVariantBis ? INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS : undefined,
catalogVersion: isVariantBis
? INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_BIS_VERSION
: INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION,
});
if (taxonomyResolution.source === 'taxonomy') {
@@ -338,6 +386,7 @@ function setPersonalizedPromptResolution(metadataLoadState: PersonalizedPromptMe
return;
}
}
// EOF Experiment cleanup
const format = personalizedPromptSuggestionsFormat.value;
if (!format) {
@@ -364,10 +413,12 @@ function resolvePersonalizedPromptMetadata() {
return;
}
if (!isInspirationFromTaxonomyTreatmentVariant.value && personalizedPromptProfileOverride.value) {
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
if (!isTaxonomyRowsVariant.value && personalizedPromptProfileOverride.value) {
setPersonalizedPromptResolution('loaded');
return;
}
// EOF Experiment cleanup
if (!appSettingsStore.isCloudDeployment) {
setPersonalizedPromptResolution('not_cloud');
@@ -387,7 +438,8 @@ function resolvePersonalizedPromptMetadata() {
watch(
[
isInspirationFromTaxonomyTreatmentVariant,
isTaxonomyRowsVariant, // Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
inspirationFromTaxonomyVariant, // Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
isPersonalizedPromptSuggestionsTreatmentVariant,
personalizedPromptSuggestionsFormat,
personalizedPromptProfileOverride,
@@ -421,6 +473,7 @@ watch(
{ immediate: true },
);
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
watch(
shouldTrackInspirationFromTaxonomyExposure,
(shouldTrackExposure) => {
@@ -434,13 +487,14 @@ watch(
}
telemetry.track(
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPOSURE_EVENT,
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
getExperimentTelemetryPayload(INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT, variant),
);
hasTrackedInspirationFromTaxonomyExposure = true;
},
{ immediate: true },
);
// EOF Experiment cleanup
// Experiment cleanup (124_workflow_previews_above_assistant)
watch(
@@ -477,11 +531,11 @@ watch(
},
);
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
const isTaxonomySuggestionsPending = computed(
() =>
isInspirationFromTaxonomyTreatmentVariant.value &&
personalizedPromptSuggestionResolution.value === null,
() => isTaxonomyRowsVariant.value && personalizedPromptSuggestionResolution.value === null,
);
// EOF Experiment cleanup
const shouldShowPersonalizedPromptSuggestions = computed(() =>
isPersonalizedPromptSuggestionResolution(personalizedPromptSuggestionResolution.value),
);
@@ -511,6 +565,7 @@ const emptyStatePromptSuggestionProps = computed(() => {
return {};
}
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
if (isTaxonomySuggestionsPending.value) {
return {
suggestions: [],
@@ -522,13 +577,9 @@ const emptyStatePromptSuggestionProps = computed(() => {
if (isTaxonomyPromptSuggestionResolution(resolution)) {
return {
suggestions: resolution.suggestions,
suggestionsComponent: InstanceAiPersonalizedPromptSuggestions,
suggestionsComponentProps: {
fallbackSuggestions: [],
format: 'list',
showSeeMore: resolution.showSeeMore,
},
suggestionCatalogVersion: INSTANCE_AI_TAXONOMY_PROMPT_SUGGESTIONS_VERSION,
suggestionsComponent: TaxonomyPromptSuggestions,
suggestionsComponentProps: {},
suggestionCatalogVersion: resolution.telemetryPayload.suggestion_catalog_version,
suggestionTelemetryPayload: getExperimentTelemetryPayload(
INSTANCE_AI_INSPIRATION_FROM_TAXONOMY_EXPERIMENT,
inspirationFromTaxonomyVariant.value,
@@ -537,6 +588,7 @@ const emptyStatePromptSuggestionProps = computed(() => {
placeholderKey: INSTANCE_AI_PERSONALIZED_PROMPT_SUGGESTIONS_PLACEHOLDER_KEY,
};
}
// EOF Experiment cleanup
if (isPersonalizedPromptSuggestionsTreatmentVariant.value) {
if (!resolution) {
@@ -588,8 +640,8 @@ const emptyStateTitleKey = computed<BaseTextKey>(() => {
return INSTANCE_AI_WORKFLOW_PREVIEW_SUGGESTIONS_TITLE_KEY;
}
if (
shouldShowTaxonomySuggestions.value ||
isTaxonomySuggestionsPending.value ||
shouldShowTaxonomySuggestions.value || // Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
isTaxonomySuggestionsPending.value || // Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
shouldShowPersonalizedPromptSuggestions.value ||
(isPersonalizedPromptSuggestionsTreatmentVariant.value &&
personalizedPromptSuggestionResolution.value === null)
@@ -14,6 +14,7 @@ import { mockedStore } from '@/__tests__/utils';
import InstanceAiEmptyView from '../InstanceAiEmptyView.vue';
import { useInstanceAiStore, type ThreadRuntime } from '../instanceAi.store';
import { useSettingsStore } from '@n8n/stores/settings.store';
import { usePostHog } from '@/app/stores/posthog.store';
import { INSTANCE_AI_THREAD_VIEW } from '../constants';
import { useProjectsStore } from '@/features/collaboration/projects/projects.store';
import type { Project, ProjectListItem } from '@/features/collaboration/projects/projects.types';
@@ -49,8 +50,10 @@ const {
workflowPreviewsAboveAssistantVariant: { value: null as 'control' | 'variant' | null },
// EOF Experiment cleanup
splitBelowInputVariant: { value: false },
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
inspirationFromTaxonomyVariant: { value: undefined as string | undefined },
inspirationFromTaxonomyTreatmentEnabled: { value: false },
// EOF Experiment cleanup
personalizedPromptVariant: { value: undefined as string | undefined },
personalizedPromptFormat: { value: null as 'cards' | 'list' | null },
personalizedPromptTreatmentEnabled: { value: false },
@@ -258,13 +261,20 @@ vi.mock('@/experiments/instanceAiPersonalizedPromptSuggestions', () => ({
resolvePersonalizedPromptSuggestions: experimentMocks.resolvePersonalizedPromptSuggestions,
}));
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
vi.mock('@/experiments/instanceAiInspirationFromTaxonomy', async (importOriginal) => ({
...(await importOriginal<object>()),
useInstanceAiInspirationFromTaxonomyExperiment: () => ({
currentVariant: experimentMocks.inspirationFromTaxonomyVariant,
isTreatmentVariant: experimentMocks.inspirationFromTaxonomyTreatmentEnabled,
isRowsVariant: computed(
() =>
experimentMocks.inspirationFromTaxonomyVariant.value === 'variant' ||
experimentMocks.inspirationFromTaxonomyVariant.value === 'variant-bis',
),
}),
}));
// EOF Experiment cleanup
vi.mock('@/experiments/instanceAiWorkflowPreviewSuggestions', () => ({
INSTANCE_AI_WORKFLOW_PREVIEW_SUGGESTIONS: workflowPreviewSuggestions,
@@ -499,6 +509,18 @@ const InstanceAiInputStub = defineComponent({
{ 'data-test-id': 'instance-ai-input-suggestions-component' },
props.suggestionsComponent === undefined ? 'unset' : 'set',
),
h(
'span',
{ 'data-test-id': 'instance-ai-input-suggestions-component-name' },
props.suggestionsComponent &&
typeof props.suggestionsComponent === 'object' &&
('name' in props.suggestionsComponent || '__name' in props.suggestionsComponent)
? String(
(props.suggestionsComponent as { name?: string; __name?: string }).name ??
(props.suggestionsComponent as { __name?: string }).__name,
)
: 'unset',
),
h(
'span',
{ 'data-test-id': 'instance-ai-input-suggestion-catalog-version' },
@@ -664,8 +686,10 @@ describe('InstanceAiEmptyView', () => {
experimentMocks.workflowPreviewsAboveAssistantVariant.value = null;
// EOF Experiment cleanup
experimentMocks.splitBelowInputVariant.value = false;
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
experimentMocks.inspirationFromTaxonomyVariant.value = undefined;
experimentMocks.inspirationFromTaxonomyTreatmentEnabled.value = false;
// EOF Experiment cleanup
experimentMocks.personalizedPromptVariant.value = undefined;
experimentMocks.personalizedPromptFormat.value = null;
experimentMocks.personalizedPromptTreatmentEnabled.value = false;
@@ -976,7 +1000,8 @@ describe('InstanceAiEmptyView', () => {
vi.useRealTimers();
});
it('shows taxonomy list suggestions and exposes the treatment for a mapped role', () => {
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
it('shows taxonomy row suggestions and exposes the variant for a sales respondent', () => {
experimentMocks.inspirationFromTaxonomyVariant.value = 'variant';
experimentMocks.inspirationFromTaxonomyTreatmentEnabled.value = true;
appSettingsStoreMock.isCloudDeployment = true;
@@ -987,17 +1012,51 @@ describe('InstanceAiEmptyView', () => {
const { getByTestId } = renderView();
expect(getByTestId('instance-ai-input-suggestions')).toHaveTextContent('4');
expect(getByTestId('instance-ai-input-suggestions')).toHaveTextContent('6');
expect(getByTestId('instance-ai-input-suggestion-catalog-version')).toHaveTextContent(
'v5-taxonomy',
'v6-taxonomy',
);
expect(getByTestId('instance-ai-input-suggestions-component-props')).toHaveTextContent(
'"format":"list"',
expect(getByTestId('instance-ai-input-suggestions-component-name')).toHaveTextContent(
'TaxonomyPromptSuggestions',
);
expect(telemetryTrack).toHaveBeenCalledWith('Instance AI inspiration from taxonomy exposed', {
variant: 'variant',
'$feature/112_aia_inspiration_from_taxonomy': 'variant',
});
expect(getByTestId('instance-ai-input-suggestion-telemetry-payload')).toHaveTextContent(
'"suggestion_format":"rows"',
);
expect(telemetryTrack).toHaveBeenCalledWith(
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
{
variant: 'variant',
'$feature/129_aia_inspiration_from_taxonomy_v2': 'variant',
},
);
expect(usePostHog().setTaxonomyInspirationEligible).toHaveBeenCalledWith(true);
});
it('shows the alternate suggestion catalog for variant-bis', () => {
experimentMocks.inspirationFromTaxonomyVariant.value = 'variant-bis';
appSettingsStoreMock.isCloudDeployment = true;
cloudPlanStoreMock.state.initialized = true;
cloudPlanStoreMock.currentUserCloudInfo = {
information: { what_team_are_you_on: 'Marketing' },
};
const { getByTestId } = renderView();
expect(getByTestId('instance-ai-input-suggestions')).toHaveTextContent('6');
expect(getByTestId('instance-ai-input-suggestion-catalog-version')).toHaveTextContent(
'v6-taxonomy-bis',
);
expect(getByTestId('instance-ai-input-suggestion-telemetry-payload')).toHaveTextContent(
'"suggestion_catalog_version":"v6-taxonomy-bis"',
);
expect(telemetryTrack).toHaveBeenCalledWith(
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
{
variant: 'variant-bis',
'$feature/129_aia_inspiration_from_taxonomy_v2': 'variant-bis',
},
);
expect(usePostHog().setTaxonomyInspirationEligible).toHaveBeenCalledWith(true);
});
it('treats an unmapped taxonomy role as control and does not expose the treatment', () => {
@@ -1015,9 +1074,31 @@ describe('InstanceAiEmptyView', () => {
'v3-workflow-preview',
);
expect(telemetryTrack).not.toHaveBeenCalledWith(
'Instance AI inspiration from taxonomy exposed',
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
expect.anything(),
);
expect(usePostHog().setTaxonomyInspirationEligible).not.toHaveBeenCalled();
});
it('excludes an IT respondent from the taxonomy suggestions and from exposure', () => {
experimentMocks.inspirationFromTaxonomyVariant.value = 'variant';
experimentMocks.inspirationFromTaxonomyTreatmentEnabled.value = true;
appSettingsStoreMock.isCloudDeployment = true;
cloudPlanStoreMock.state.initialized = true;
cloudPlanStoreMock.currentUserCloudInfo = {
information: { what_team_are_you_on: 'IT' },
};
const { getByTestId } = renderView();
expect(getByTestId('instance-ai-input-suggestion-catalog-version')).toHaveTextContent(
'v3-workflow-preview',
);
expect(telemetryTrack).not.toHaveBeenCalledWith(
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
expect.anything(),
);
expect(usePostHog().setTaxonomyInspirationEligible).not.toHaveBeenCalled();
});
it('waits for taxonomy metadata before falling back to the control catalog', async () => {
@@ -1056,7 +1137,7 @@ describe('InstanceAiEmptyView', () => {
const { getByTestId } = renderView();
expect(getByTestId('instance-ai-input-suggestion-catalog-version')).toHaveTextContent(
'v5-taxonomy',
'v6-taxonomy',
);
expect(experimentMocks.resolvePersonalizedPromptSuggestions).not.toHaveBeenCalled();
});
@@ -1080,6 +1161,7 @@ describe('InstanceAiEmptyView', () => {
);
expect(experimentMocks.resolvePersonalizedPromptSuggestions).toHaveBeenCalled();
});
// EOF Experiment cleanup
it('passes workflow preview suggestions, component, and catalog version', () => {
const { getByTestId, getByText } = renderView();
@@ -1357,7 +1439,8 @@ describe('InstanceAiEmptyView', () => {
});
// EOF Experiment cleanup
it('tracks inspiration from taxonomy exposure for a control user with a mapped role', () => {
// Experiment cleanup (129_aia_inspiration_from_taxonomy_v2)
it('keeps the default empty state for a sales-and-marketing control user and still records exposure', () => {
experimentMocks.inspirationFromTaxonomyVariant.value = 'control';
appSettingsStoreMock.isCloudDeployment = true;
cloudPlanStoreMock.state.initialized = true;
@@ -1365,12 +1448,19 @@ describe('InstanceAiEmptyView', () => {
information: { what_team_are_you_on: 'Sales' },
};
renderView();
const { getByTestId } = renderView();
expect(telemetryTrack).toHaveBeenCalledWith('Instance AI inspiration from taxonomy exposed', {
variant: 'control',
'$feature/112_aia_inspiration_from_taxonomy': 'control',
});
expect(getByTestId('instance-ai-input-suggestion-catalog-version')).toHaveTextContent(
'v3-workflow-preview',
);
expect(telemetryTrack).toHaveBeenCalledWith(
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
{
variant: 'control',
'$feature/129_aia_inspiration_from_taxonomy_v2': 'control',
},
);
expect(usePostHog().setTaxonomyInspirationEligible).toHaveBeenCalledWith(true);
});
it('does not track inspiration from taxonomy exposure for a control user without a mapped role', () => {
@@ -1384,9 +1474,10 @@ describe('InstanceAiEmptyView', () => {
renderView();
expect(telemetryTrack).not.toHaveBeenCalledWith(
'Instance AI inspiration from taxonomy exposed',
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
expect.anything(),
);
expect(usePostHog().setTaxonomyInspirationEligible).not.toHaveBeenCalled();
});
it.each([
@@ -1402,30 +1493,43 @@ describe('InstanceAiEmptyView', () => {
experimentMocks.proactiveAgentEnabled.value = true;
},
},
// Experiment cleanup (124_workflow_previews_above_assistant)
{
name: 'the workflow previews layout',
setup: () => {
experimentMocks.workflowPreviewsAboveAssistantEnabled.value = true;
},
])(
'does not track inspiration from taxonomy exposure when $name replaces the empty state',
({ setup }) => {
experimentMocks.inspirationFromTaxonomyVariant.value = 'control';
appSettingsStoreMock.isCloudDeployment = true;
cloudPlanStoreMock.state.initialized = true;
cloudPlanStoreMock.currentUserCloudInfo = {
information: { what_team_are_you_on: 'Sales' },
};
setup();
renderView();
expect(telemetryTrack).not.toHaveBeenCalledWith(
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
expect.anything(),
);
},
// EOF Experiment cleanup
])('does not track inspiration from taxonomy exposure when $name is active', ({ setup }) => {
);
it('does not record taxonomy exposure when workflow previews replace the suggestion list', () => {
experimentMocks.inspirationFromTaxonomyVariant.value = 'control';
experimentMocks.workflowPreviewsAboveAssistantEnabled.value = true;
appSettingsStoreMock.isCloudDeployment = true;
cloudPlanStoreMock.state.initialized = true;
cloudPlanStoreMock.currentUserCloudInfo = {
information: { what_team_are_you_on: 'Sales' },
};
setup();
renderView();
expect(telemetryTrack).not.toHaveBeenCalledWith(
'Instance AI inspiration from taxonomy exposed',
TELEMETRY_EVENT.INSTANCE_AI.INSPIRATION_FROM_TAXONOMY_EXPOSED,
expect.anything(),
);
});
// EOF Experiment cleanup
it('does not track personalized prompt suggestions exposure when workflow builder is unavailable', () => {
experimentMocks.personalizedPromptVariant.value = 'control';
@@ -159,7 +159,12 @@ describe('instance ai prompt suggestions telemetry', () => {
threadId: 'thread-abc',
suggestionId: 'build-workflow',
suggestionKind: 'prompt',
suggestionTitle: 'Weekly report on my ads and website, with what changed',
position: 1,
telemetryPayload: {
variant: 'variant',
'$feature/129_aia_inspiration_from_taxonomy_v2': 'variant',
},
});
expect(track).toHaveBeenCalledWith('Instance AI prompt suggestion selected', {
@@ -167,7 +172,10 @@ describe('instance ai prompt suggestions telemetry', () => {
suggestion_catalog_version: INSTANCE_AI_EMPTY_STATE_SUGGESTIONS_VERSION,
suggestion_id: 'build-workflow',
suggestion_kind: 'prompt',
suggestion_title: 'Weekly report on my ads and website, with what changed',
position: 1,
variant: 'variant',
'$feature/129_aia_inspiration_from_taxonomy_v2': 'variant',
});
const selectedPayload = track.mock.calls[0][1];
@@ -58,6 +58,7 @@ export type SuggestionSelectionPayload = SuggestionPromptPayload & {
telemetryPayload?: ITelemetryTrackProperties;
/** Required so a new catalog cannot emit suggestions that report as user-typed. */
prefillType: InstanceAiPrefillType;
suggestionTitle?: string;
/**
* Catalog this row belongs to. A host-triggered submit (agent templates)
* is not the home-screen catalog mounted on this input.
@@ -866,6 +867,7 @@ function trackSuggestionSelected(payload: SuggestionSelectionPayload) {
suggestionId: payload.suggestionId,
suggestionKind: payload.suggestionKind,
position: payload.position,
suggestionTitle: payload.suggestionTitle,
});
}
@@ -28,6 +28,7 @@ export type InstanceAiPromptSuggestionSelectedContext = InstanceAiPromptSuggesti
suggestionId: string;
suggestionKind: 'prompt' | 'quick_example';
position: number;
suggestionTitle?: string;
};
export type InstanceAiPromptSuggestionSubmittedContext =
@@ -99,6 +100,7 @@ export function createInstanceAiPromptSuggestionsTelemetry(
suggestion_id: context.suggestionId,
suggestion_kind: context.suggestionKind,
position: context.position,
...(context.suggestionTitle ? { suggestion_title: context.suggestionTitle } : {}),
});
},