fix(pi): surface provider errors from agent events (#6133)

Co-authored-by: Bill Leoutsakos <billleoutsakos@Bills-MacBook-Pro.local>

apps/sim/executor/handlers/pi/cloud-backend.test.ts

@@ -532,7 +532,43 @@ describe('runCloudPi', () => {
           return Promise.resolve({ stdout: '__BASE_SHA__=abc', stderr: '', exitCode: 0 })
         }
         if (command.includes('pi -p')) {
-          options.onStdout?.('{"type":"error","error":"model exploded"}\n')
+          options.onStdout?.(
+            `${[
+              JSON.stringify({
+                type: 'message_end',
+                message: {
+                  role: 'assistant',
+                  content: [{ type: 'text', text: '' }],
+                  usage: { input: 0, output: 0, totalTokens: 0 },
+                  stopReason: 'error',
+                  errorMessage: 'model rejected sk-byok',
+                },
+              }),
+              JSON.stringify({
+                type: 'turn_end',
+                message: {
+                  role: 'assistant',
+                  usage: { input: 0, output: 0, totalTokens: 0 },
+                  stopReason: 'error',
+                  errorMessage: 'model rejected sk-byok',
+                },
+                toolResults: [],
+              }),
+              JSON.stringify({
+                type: 'agent_end',
+                willRetry: false,
+                messages: [
+                  {
+                    role: 'assistant',
+                    content: [{ type: 'text', text: '' }],
+                    usage: { input: 0, output: 0, totalTokens: 0 },
+                    stopReason: 'error',
+                    errorMessage: 'model rejected sk-byok',
+                  },
+                ],
+              }),
+            ].join('\n')}\n`
+          )
           return Promise.resolve({ stdout: '', stderr: '', exitCode: 0 })
         }
         return Promise.resolve({
@@ -543,9 +579,11 @@ describe('runCloudPi', () => {
       }
     )
 
-    await expect(runCloudPi(baseParams(), { onEvent: vi.fn() })).rejects.toThrow(/model exploded/)
+    await expect(runCloudPi(baseParams(), { onEvent: vi.fn() })).rejects.toThrow(
+      'model rejected ***'
+    )
+    expect(mockRun).toHaveBeenCalledTimes(2)
     expect(mockExecuteTool).not.toHaveBeenCalled()
-    expect(mockRun.mock.calls.some(([cmd]: [string]) => cmd.includes('push'))).toBe(false)
   })
 
   it('fails (no PR) when finalize reports neither no-changes nor a push', async () => {

apps/sim/executor/handlers/pi/events.test.ts

@@ -52,8 +52,51 @@ describe('normalizePiEvent', () => {
     ).toEqual({ type: 'usage', inputTokens: 3, outputTokens: 2 })
   })
 
+  it('maps a settled agent failure to an error', () => {
+    expect(
+      normalizePiEvent({
+        type: 'agent_end',
+        willRetry: false,
+        messages: [
+          {
+            role: 'assistant',
+            stopReason: 'error',
+            errorMessage: 'Invalid API key',
+          },
+        ],
+      })
+    ).toEqual({ type: 'error', message: 'Invalid API key' })
+    expect(
+      normalizePiEvent({
+        type: 'agent_end',
+        messages: [{ role: 'assistant', stopReason: 'aborted' }],
+      })
+    ).toEqual({ type: 'error', message: 'Pi request aborted' })
+  })
+
+  it('does not fail an attempt that Pi will retry', () => {
+    expect(
+      normalizePiEvent({
+        type: 'agent_end',
+        willRetry: true,
+        messages: [
+          {
+            role: 'assistant',
+            stopReason: 'error',
+            errorMessage: 'Provider overloaded',
+          },
+        ],
+      })
+    ).toEqual({ type: 'other' })
+  })
+
   it('maps agent_end to final and error to error', () => {
-    expect(normalizePiEvent({ type: 'agent_end' })).toEqual({ type: 'final' })
+    expect(
+      normalizePiEvent({
+        type: 'agent_end',
+        messages: [{ role: 'assistant', stopReason: 'stop' }],
+      })
+    ).toEqual({ type: 'final' })
     expect(normalizePiEvent({ type: 'error', error: 'boom' })).toEqual({
       type: 'error',
       message: 'boom',

apps/sim/executor/handlers/pi/events.ts

@@ -136,8 +136,23 @@ export function normalizePiEvent(raw: unknown): PiEvent | null {
       const usage = extractUsage(ev)
       return usage ? { type: 'usage', ...usage } : { type: 'other' }
     }
-    case 'agent_end':
+    case 'agent_end': {
+      if (ev.willRetry === true) return { type: 'other' }
+      const messages = Array.isArray(ev.messages) ? ev.messages : []
+      for (let index = messages.length - 1; index >= 0; index -= 1) {
+        const message = asRecord(messages[index])
+        if (!message || asString(message.role) !== 'assistant') continue
+        const stopReason = asString(message.stopReason)
+        if (stopReason === 'error' || stopReason === 'aborted') {
+          return {
+            type: 'error',
+            message: asString(message.errorMessage) || `Pi request ${stopReason}`,
+          }
+        }
+        break
+      }
       return { type: 'final' }
+    }
     case 'error':
       return {
         type: 'error',