From e804cd418aa9bacb83eb94eeab37a3925bcb110e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 7 Nov 2026 11:24:32 +0900 Subject: [PATCH 01/26] chore(release): open dev at 2.99.0 before releasing 2.98.0 (#8263) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- desktop/src-tauri/Cargo.lock | 2 +- desktop/src-tauri/Cargo.toml | 2 +- desktop/src-tauri/tauri.conf.json | 2 +- package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index 960e6146bcf..ddc9ff16b55 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -2633,7 +2633,7 @@ dependencies = [ [[package]] name = "opencodex-desktop" -version = "2.98.0" +version = "2.99.0" dependencies = [ "base64 0.22.1", "dbus", diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 9707db5345a..201998c2a13 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opencodex-desktop" -version = "2.98.0" +version = "2.99.0" description = "OpenCodex desktop shell" authors = ["OpenCodex contributors"] license = "MIT" diff --git a/desktop/src-tauri/tauri.conf.json b/desktop/src-tauri/tauri.conf.json index 544a26fe31b..892d5c37df0 100644 --- a/desktop/src-tauri/tauri.conf.json +++ b/desktop/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "OpenCodex", - "version": "2.98.0", + "version": "2.99.0", "identifier": "com.opencodex.desktop", "build": { "frontendDist": "../ui", diff --git a/package.json b/package.json index 74afddd11e8..52c50ccd7f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@bitkyc08/opencodex", - "version": "2.98.0", + "version": "2.99.0", "description": "Universal provider proxy for OpenAI Codex & Claude Code — use any LLM with Codex CLI/App/SDK and Claude Code", "type": "module", "main": "./bin/package-main.mjs", From afd76e0b8117433348a6db40609588057d31dd42 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 7 Nov 2026 12:14:25 +0900 Subject: [PATCH 02/26] fix(mistral): ship per-model effort ladders and fold native Chat ultra (#8259) Mistral exports list only accepted reasoning_effort rungs per model, and native Chat folds caller ultra through mapReasoningEffort / ladder clamp. Fixes #8254 Co-authored-by: Masih-0x3 (Steve) --- .../src/content/docs/guides/providers.md | 26 +++++ scripts/test-layout/layout.json | 2 +- src/adapters/openai-chat/passthrough.ts | 8 +- src/adapters/openai-chat/reasoning-wire.ts | 19 ++++ src/providers/registry/entries-extended.ts | 13 +++ structure/catalog.md | 11 +-- .../model-pinned-effort.test.ts | 4 +- tests/fixtures/test-layout-expected.json | 2 +- .../mistral-reasoning-efforts.test.ts | 97 +++++++++++++++++++ 9 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 tests/providers/mistral-reasoning-efforts.test.ts diff --git a/docs-site/src/content/docs/guides/providers.md b/docs-site/src/content/docs/guides/providers.md index a6cba2299fc..a2bdc319a95 100644 --- a/docs-site/src/content/docs/guides/providers.md +++ b/docs-site/src/content/docs/guides/providers.md @@ -162,6 +162,32 @@ disables shared pacing. If Mistral's retry time exceeds `retryOn429.maxIntervalMs`, the proxy returns the 429 and retry hint rather than replaying early. Context windows and per-model compaction preferences remain independent. +Mistral accepts a different set of `reasoning_effort` values for each model and refuses the +rest with HTTP 400 (or 422 for `ultra`). The preset ships the accepted rungs as registry +`modelReasoningEfforts` rows. Client exports such as `ocx export --client opencode` offer only +these variants. The Codex catalog still adds its synthetic `ultra` rung, plus `max` where a +model's list omits it, and requests clamp those to the highest listed rung: + +| Model | Efforts | +|---|---| +| `zai-glm-5-3`, `zai-glm-5-3-flash` | `low`, `high`, `max` | +| `mistral-large-4`, `mistral-large-4-0` | `low`, `medium`, `high` | +| `deepseek-v4-1-flash-1017` | `low`, `high`, `xhigh`, `max` | + +The GLM and Large 4 rows follow [Mistral's reasoning guide](https://docs.mistral.ai/capabilities/reasoning). +Mistral documents no values for DeepSeek V4.1 Flash 1017, so its row is the set measured to +answer HTTP 200 in November 2026 and may change. A `modelReasoningEfforts` entry of your own +for a model replaces its row. + +On native Chat Completions, for a model with reasoning enabled, a caller's `ultra` is sent as `max`, or as the model's highest +declared rung below `max` (`high` on Large 4). This applies to every `openai-chat` provider +unless the model's declared ladder lists `ultra` itself. A `reasoningEffortMap` alias for `max` +applies to the folded value, and `__omit__` drops the field, except on thinking-toggle models, +where the fold skips the alias map ([#8266](https://github.com/lidge-jun/opencodex/issues/8266)). Models with reasoning disabled (for +example `noReasoningModels`) drop `reasoning_effort` entirely. Other caller values are forwarded +as sent unless an effort pin or a qualifying effort cap rewrites them; rewritten values also go +through the provider's wire mapping. + When round-robin key selection is enabled, queued requests recheck key cooldowns before sending and acquire pacing capacity for any replacement key. This applies to native Chat, Messages, Responses and compact. Waiting for replacement-key capacity does not consume the upstream response-header timeout. Per-key pacing and round-robin use guarded HTTP/SSE even when upstream WebSockets are enabled; WebSocket-only steering and injection require a route without those key policies. This does not change the saved active key. An established upstream turn keeps its credential; later sends are refused if that key cools. Pre-output key rejection can recover to a healthy key within the existing request limit. Direct Images and compact requests record genuine 401/429 cooldowns without replaying that paid POST. ### Anthropic API-key credit exhaustion diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index a7dd71a9be1..c493d1db6f5 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1469,7 +1469,7 @@ "qwen-cloud-endpoints.test.ts": "gui", "qwen38-preserve-reasoning.test.ts": "providers", "rate-limit-reset-credits.test.ts": "gui", - "rate-limit-retry.test.ts": "providers", "mistral-recovery.test.ts": "providers", + "rate-limit-retry.test.ts": "providers", "mistral-recovery.test.ts": "providers", "mistral-reasoning-efforts.test.ts": "providers", "reasoning-effort-summary-default.test.ts": "responses", "reasoning-effort.test.ts": "codex-integration", "reasoning-envelope.test.ts": "responses", diff --git a/src/adapters/openai-chat/passthrough.ts b/src/adapters/openai-chat/passthrough.ts index e6d047f202c..3ed1622c6db 100644 --- a/src/adapters/openai-chat/passthrough.ts +++ b/src/adapters/openai-chat/passthrough.ts @@ -16,7 +16,8 @@ import { modelInList, type OcxProviderConfig } from "../../types"; import { mapReasoningEffort, modelRecordValue } from "../../reasoning-effort"; import { resolveReasoningCapabilities } from "../../providers/reasoning-capabilities"; import { chatParallelToolCallsWireValue } from "./parallel-tool-calls"; -import { applyExplicitChatReasoningWirePolicy, declaresIndependentChatReasoningControls } from "./reasoning-wire"; +import { applyExplicitChatReasoningWirePolicy, declaresIndependentChatReasoningControls, + nativeChatUltraWireEffort } from "./reasoning-wire"; import { declaresThinkingToggleWire, reasoningControlsAreUnsupported } from "../reasoning-controls"; import { planNativeChatReasoningReplay } from "./reasoning-replay"; import type { OcxReasoningReplayScopeRef } from "../../types"; @@ -101,6 +102,11 @@ export function buildOpenAIChatPassthroughRequest( if (reasoningDisabled) { delete body.reasoning_effort; } + if (body.reasoning_effort === "ultra") { + const folded = nativeChatUltraWireEffort(provider, modelId); + if (folded === undefined) delete body.reasoning_effort; + else body.reasoning_effort = folded; + } const hasTools = Array.isArray(rawBody.tools) && rawBody.tools.length > 0; const requestedEffort = typeof body.reasoning_effort === "string" ? body.reasoning_effort : undefined; const explicitThinkingEnabled = rawBody.reasoning !== null && typeof rawBody.reasoning === "object" diff --git a/src/adapters/openai-chat/reasoning-wire.ts b/src/adapters/openai-chat/reasoning-wire.ts index c63c48a3491..f87650b5bc2 100644 --- a/src/adapters/openai-chat/reasoning-wire.ts +++ b/src/adapters/openai-chat/reasoning-wire.ts @@ -1,6 +1,7 @@ import { resolveThinkingEnabled } from "../../clients/reasoning-controls"; import { declaredModelCapabilities, resolveReasoningCapabilities } from "../../providers/reasoning-capabilities"; import { registryEntryForProviderDestination } from "../../providers/registry"; +import { configuredReasoningEfforts, mapReasoningEffort, resolveEffortAtOrBelow } from "../../reasoning-effort"; import { modelInList, type OcxProviderConfig } from "../../types"; import type { AdapterRequest } from "../base"; import { declaresThinkingToggleWire, reasoningControlsAreUnsupported } from "../reasoning-controls"; @@ -29,6 +30,24 @@ export function declaresIndependentChatReasoningControls(provider: OcxProviderCo .some(axis => typeof (explicit?.[axis] ?? registry?.[axis]) === "boolean"); } +/** + * Wire value for a native Chat caller's `ultra`. It is a Codex picker level, not an upstream + * Chat value: codex-rs folds it to `max` before every provider request and the translated path + * does the same in `mapReasoningEffort`, but OpenCode and Kilo send it verbatim, and Mistral + * answers it with HTTP 422 (#8254). The fold lands on `max`, clamped at or below it to the + * model's declared ladder (Mistral Large 4 tops out at `high`). A declared ladder that lists + * `ultra` itself is the provider's own claim and keeps it. Like a pin or cap rewrite, the folded + * value goes through the provider's wire alias map; `undefined` means omit the field. + * A thinking-toggle wire is the exception: its map holds thinking discriminators, which + * `applyExplicitChatReasoningWirePolicy` resolves itself, so it receives the plain strength. + */ +export function nativeChatUltraWireEffort(provider: OcxProviderConfig, modelId: string): string | undefined { + const ladder = configuredReasoningEfforts(provider, modelId); + if (ladder?.includes("ultra")) return "ultra"; + if (declaresThinkingToggleWire(provider, modelId)) return ladder ? resolveEffortAtOrBelow("max", ladder) ?? "max" : "max"; + return mapReasoningEffort(provider, modelId, "ultra"); +} + /** * Apply provider-declared reasoning wire policy after the effective provider is resolved. * Unset declarations are a no-op so native Chat keeps forwarding the caller's raw field. diff --git a/src/providers/registry/entries-extended.ts b/src/providers/registry/entries-extended.ts index 8b3a48f381a..3d66688c9e4 100644 --- a/src/providers/registry/entries-extended.ts +++ b/src/providers/registry/entries-extended.ts @@ -1055,6 +1055,19 @@ export const PROVIDER_REGISTRY_EXTENDED: readonly ProviderRegistryEntry[] = [ "mistral-large-4-0": { mistralMaxImages: 100 }, "zai-glm-5-3-flash": { mistralMaxImages: 256 }, }, + // Effort rungs Mistral accepts per model (#8254); without them exports offer the whole + // low..ultra ladder and Mistral refuses medium/xhigh with 400 and ultra with 422. GLM 5.3, + // GLM 5.3 Flash and Large 4 follow docs.mistral.ai/capabilities/reasoning. DeepSeek V4.1 + // Flash 1017 has no documented values: its rungs are the reporter's measured HTTP 200 set + // (2026-11), every one inside Mistral's OpenAPI ReasoningEffort enum; medium and ultra were + // measured refusals. An operator modelReasoningEfforts entry still replaces any row here. + modelReasoningEfforts: { + "zai-glm-5-3": ["low", "high", "max"], + "zai-glm-5-3-flash": ["low", "high", "max"], + "mistral-large-4": ["low", "medium", "high"], + "mistral-large-4-0": ["low", "medium", "high"], + "deepseek-v4-1-flash-1017": ["low", "high", "xhigh", "max"], + }, }, { id: "minimax", label: "MiniMax — Coding Plan", baseUrl: "https://api.minimax.io/v1", adapter: "openai-chat", authKind: "key", diff --git a/structure/catalog.md b/structure/catalog.md index 7d86eff1277..f9193633c27 100644 --- a/structure/catalog.md +++ b/structure/catalog.md @@ -486,7 +486,7 @@ Catalog rows retain independent reasoning facts from [per-model declarations](co `src/codex/catalog/effort.ts` uses the [client export projection](clients/integrations.md#model-output-limits) at Codex's effort-enum boundary: toggle-only models receive `none/high` off/on choices and a declared thinking default, without recording those choices as model strengths. Explicit toggle-axis declarations or negative reasoning/effort declarations constrain this projection and suppress synthetic top tiers. Positive strength support inferred from a legacy ladder alone retains its previous synthesis. -Legacy catalog ladders advertise ultra regardless of the `multi_agent_v2` toggle, which controls only the multi-agent collab surface. `nativeEffortClamp` wire-clamps ultra/max to each model's real top rung (e.g. gpt-5.5 ultra → xhigh on the wire). `ultra` itself is never a native wire value: the ChatGPT backend accepts `none` through `max`, so the clamp folds it to `max` even on rows whose pinned ladder lists it (`gpt-6-astra`, `gpt-6.1-sol`, `gpt-6.1-terra`). `src/server/responses/core-normalize.ts` also applies the clamp to the raw passthrough body, which the parser's options-only fold does not reach (#7590; `tests/responses/responses-native-ultra-effort-wire.test.ts`). For canonical OpenAI forwarding, other unsupported efforts map to the nearest published rung at or below the request, or the lowest rung when below the model minimum (`gpt-6.1-sol` and `gpt-6.1-terra` `none` → `low`), after final model selection, including combo failover. +Legacy catalog ladders advertise ultra regardless of the `multi_agent_v2` toggle, which controls only the multi-agent collab surface. `nativeEffortClamp` wire-clamps ultra/max to each model's real top rung (e.g. gpt-5.5 ultra → xhigh on the wire). `ultra` itself is never a native ChatGPT-backend wire value: that backend accepts `none` through `max`, so the clamp folds it to `max` even on rows whose pinned ladder lists it (`gpt-6-astra`, `gpt-6.1-sol`, `gpt-6.1-terra`). `src/server/responses/core-normalize.ts` also applies the clamp to the raw passthrough body, which the parser's options-only fold does not reach (#7590; `tests/responses/responses-native-ultra-effort-wire.test.ts`). The routed native Chat passthrough does the same for a caller's raw `ultra`: `nativeChatUltraWireEffort` in `src/adapters/openai-chat/reasoning-wire.ts` keeps `ultra` only when the model's declared ladder lists it. Otherwise it resolves `max` through `mapReasoningEffort` like the translated path: an operator alias for `max` wins before the ladder clamp (so a `max` → `ultra` alias still sends `ultra`, and `__omit__` drops the field), then the clamp picks the highest declared rung at or below `max`. On a thinking-toggle wire it skips the alias map and clamps `max` to the ladder directly, because that map holds thinking discriminators that `applyExplicitChatReasoningWirePolicy` resolves (#8254; `tests/providers/mistral-reasoning-efforts.test.ts`). For canonical OpenAI forwarding, other unsupported efforts map to the nearest published rung at or below the request, or the lowest rung when below the model minimum (`gpt-6.1-sol` and `gpt-6.1-terra` `none` → `low`), after final model selection, including combo failover. For routed models retaining legacy ladder synthesis, `modelSuppressSyntheticMax` is a catalog-only per-model setting. A true value prevents `src/codex/catalog/effort.ts` from adding a missing synthetic `max` and prevents @@ -513,11 +513,10 @@ Operator-owned `pinnedReasoningEffort`, `modelPinnedReasoningEfforts`, and root Provider model pins precede provider-wide pins, then global selector/destination pins. A pin can raise the effective caller effort; the later cap can still lower or omit it. On legacy effort paths, `none` means explicit-effort omission (provider default), not guaranteed reasoning disablement. For a declared thinking toggle, the client compatibility decoder resolves `none` to thinking off before strength policy; an independent `thinkingEnabled` boolean takes precedence. Effort caps do not clear that switch, and a destination change restores the original thinking choice alongside effort before normalization. -Compaction maintenance is exempt. Pins are user overlays and do not alter registry seeds, -model discovery or advertised ladders. Native Chat applies qualifying caps even without a -pin, and normalizes pinned values and values rewritten by a cap through provider wire -mapping. Without a pin or a cap rewrite, native caller values retain their original wire -spelling; the V1 and compaction cap exemptions are preserved. +Compaction maintenance is exempt. Pins are user overlays and do not alter registry seeds, model discovery or advertised ladders. +Native Chat applies qualifying caps even without a pin, and normalizes pinned values and values rewritten by a cap through provider wire +mapping. Without a pin or a cap rewrite, native caller values retain their original wire spelling, except a caller `ultra`, +which the passthrough folds as described under the `nativeChatUltraWireEffort` contract above; the V1 and compaction cap exemptions are preserved. > Decision record: [ADR-0023](decisions/ADR-0023-ultra-reasoning-level.md) diff --git a/tests/codex-integration/model-pinned-effort.test.ts b/tests/codex-integration/model-pinned-effort.test.ts index 9aa8063f57b..0e3ddb69f1d 100644 --- a/tests/codex-integration/model-pinned-effort.test.ts +++ b/tests/codex-integration/model-pinned-effort.test.ts @@ -438,7 +438,9 @@ describe("operator pins on the actual request wire", () => { test("native Chat without pins caps spawned children and preserves unqualified caller spelling", async () => { const c = config({ reasoningEfforts: ["low"], reasoningEffortMap: { max: "enabled" } }, { effortCap: "low", subagentEffortCap: "low" }); expect((await request(c, "chat", { reasoning_effort: "ultra" }, { "x-openai-subagent": "collab_spawn" })).reasoning_effort).toBe("low"); - expect((await request(c, "chat", { reasoning_effort: "ultra" })).reasoning_effort).toBe("ultra"); + // `ultra` is never a Chat wire value: it folds like the translated path, here through the max alias (#8254). + expect((await request(c, "chat", { reasoning_effort: "ultra" })).reasoning_effort).toBe("enabled"); + expect((await request(c, "chat", { reasoning_effort: "high" })).reasoning_effort).toBe("high"); expect(Object.hasOwn(await request(c, "chat", { reasoning_effort: undefined }), "reasoning_effort")).toBe(false); }); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 801b3c2155b..eb1492094d5 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -1209,7 +1209,7 @@ "quota-scoring.test.ts": "usage", "qwen-cloud-endpoints.test.ts": "gui", "qwen38-preserve-reasoning.test.ts": "providers", "rate-limit-reset-credits.test.ts": "gui", "rate-limit-retry.test.ts": "providers", - "mistral-recovery.test.ts": "providers", "raycast-client.test.ts": "clients", + "mistral-recovery.test.ts": "providers", "mistral-reasoning-efforts.test.ts": "providers", "raycast-client.test.ts": "clients", "raycast-detect.test.ts": "clients", "reasoning-effort-summary-default.test.ts": "responses", "reasoning-effort.test.ts": "codex-integration", "reasoning-envelope.test.ts": "responses", "reasoning-retention.test.ts": "responses", "reasoning-retention-quota.test.ts": "responses", "reasoning-archive-http.test.ts": "responses", "reasoning-archive-ws.test.ts": "responses", "reasoning-retention-routing.test.ts": "responses", "reasoning-retention-v2.test.ts": "responses", diff --git a/tests/providers/mistral-reasoning-efforts.test.ts b/tests/providers/mistral-reasoning-efforts.test.ts new file mode 100644 index 00000000000..83a8a9b59cb --- /dev/null +++ b/tests/providers/mistral-reasoning-efforts.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test"; +import { buildOpenAIChatPassthroughRequest } from "../../src/adapters/openai-chat/passthrough"; +import { buildClientConfig, type OpencodeGeneratedConfig } from "../../src/clients/config-export"; +import { applyProviderConfigHints } from "../../src/codex/catalog/model-hints"; +import { enrichProviderFromRegistry } from "../../src/providers/derive"; +import type { OcxProviderConfig } from "../../src/types"; + +/** The configuration from #8254: the Mistral preset with no effort declarations of its own. */ +function mistral(overrides: Partial = {}): OcxProviderConfig { + const provider: OcxProviderConfig = { + adapter: "openai-chat", baseUrl: "https://api.mistral.ai/v1", apiKey: "test-mistral-efforts", + defaultModel: "codestral-latest", modelInputModalities: { "zai-glm-5-3": ["text"] }, ...overrides, + }; + enrichProviderFromRegistry("mistral", provider); + return provider; +} + +/** Enabled OpenCode V1 variant keys and V2 variant ids exported for one Mistral model. */ +function exportedVariants(provider: OcxProviderConfig, id: string) { + const row = applyProviderConfigHints("mistral", provider, { provider: "mistral", id }); + const namespaced = `mistral/${id}`; + const config = buildClientConfig("opencode", { + baseUrl: "http://127.0.0.1:10100/v1", models: [{ ...row, namespaced }], + }) as OpencodeGeneratedConfig; + const v1 = config.provider.opencodex!.models[namespaced]!.variants ?? {}; + return { + v1: Object.entries(v1).filter(([, value]) => !("disabled" in value)).map(([key]) => key), + v2: config.providers.opencodex!.models[namespaced]!.variants.map(variant => variant.id), + }; +} + +/** The `reasoning_effort` a native Chat passthrough request puts on the upstream wire. */ +function wireEffort(provider: OcxProviderConfig, modelId: string, effort: string): unknown { + const request = buildOpenAIChatPassthroughRequest(provider, { + model: modelId, messages: [{ role: "user", content: "Reply with OK" }], reasoning_effort: effort, + }, modelId, false); + return (JSON.parse(request.body as string) as Record).reasoning_effort; +} + +describe("Mistral reasoning effort ladders (#8254)", () => { + test("exports offer only the effort rungs Mistral accepts for each model", () => { + const provider = mistral(); + const expected: Record = { + "zai-glm-5-3": ["low", "high", "max"], + "zai-glm-5-3-flash": ["low", "high", "max"], + "mistral-large-4": ["low", "medium", "high"], + "mistral-large-4-0": ["low", "medium", "high"], + "deepseek-v4-1-flash-1017": ["low", "high", "xhigh", "max"], + }; + for (const [id, efforts] of Object.entries(expected)) { + expect(exportedVariants(provider, id)).toEqual({ v1: efforts, v2: efforts }); + } + }); + + test("an operator ladder for a model still replaces the shipped one", () => { + const provider = mistral({ modelReasoningEfforts: { "zai-glm-5-3": ["low", "medium", "high"] } }); + expect(exportedVariants(provider, "zai-glm-5-3").v2).toEqual(["low", "medium", "high"]); + expect(exportedVariants(provider, "zai-glm-5-3-flash").v2).toEqual(["low", "high", "max"]); + }); + + test("native Chat folds ultra to the highest declared rung at or below max", () => { + const provider = mistral(); + expect(wireEffort(provider, "zai-glm-5-3", "ultra")).toBe("max"); + expect(wireEffort(provider, "deepseek-v4-1-flash-1017", "ultra")).toBe("max"); + expect(wireEffort(provider, "mistral-large-4", "ultra")).toBe("high"); + // No declared ladder: ultra is still never a Chat wire value. + expect(wireEffort(provider, "codestral-latest", "ultra")).toBe("max"); + }); + + test("native Chat keeps every other caller value and a declared ultra rung as sent", () => { + const provider = mistral(); + expect(wireEffort(provider, "zai-glm-5-3", "max")).toBe("max"); + expect(wireEffort(provider, "zai-glm-5-3", "high")).toBe("high"); + const declaresUltra: OcxProviderConfig = { + adapter: "openai-chat", baseUrl: "https://gateway.example/v1", apiKey: "test-gateway", + modelReasoningEfforts: { "frontier": ["low", "high", "max", "ultra"] }, + }; + expect(wireEffort(declaresUltra, "frontier", "ultra")).toBe("ultra"); + expect(wireEffort(declaresUltra, "other", "ultra")).toBe("max"); + }); + + test("a folded ultra goes through the provider's wire alias map, including omission", () => { + const gateway = (reasoningEffortMap: Record): OcxProviderConfig => ({ + adapter: "openai-chat", baseUrl: "https://gateway.example/v1", apiKey: "test-gateway", reasoningEffortMap, + }); + expect(wireEffort(gateway({ max: "high" }), "any", "ultra")).toBe("high"); + expect(wireEffort(gateway({ max: "__omit__" }), "any", "ultra")).toBeUndefined(); + // The alias wins before the ladder clamp: clamp-first would turn `max` into `high` and keep the field. + const ladderedOmit = { ...gateway({ max: "__omit__" }), reasoningEfforts: ["low", "high"] }; + expect(wireEffort(ladderedOmit, "any", "ultra")).toBeUndefined(); + // A `max` → `ultra` alias heals `ultra` into the ladder, so the caller's `ultra` is kept. + const aliasedUltra = { ...gateway({ max: "ultra" }), reasoningEfforts: ["low", "high", "max"] }; + expect(wireEffort(aliasedUltra, "any", "ultra")).toBe("ultra"); + // Caller values the proxy did not rewrite keep their spelling. + expect(wireEffort(gateway({ max: "high" }), "any", "max")).toBe("max"); + }); +}); From e838c30f2cbc560d9ac347c45e23165611755478 Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 7 Nov 2026 12:50:03 +0900 Subject: [PATCH 03/26] test: bound Windows overflow slices and preserve child cleanup (#8229) Co-authored-by: JUN <243035832+lidge-jun@users.noreply.github.com> --- scripts/ci/test-durations.windows.tsv | 9 +- ...essages-overflow-baseline-contract.test.ts | 166 +++++++++++++++++- tests/helpers/messages-overflow-assertions.ts | 38 ++-- tests/helpers/messages-overflow-child.ts | 98 +++++++++++ .../helpers/messages-overflow-differential.ts | 18 +- .../messages-overflow-spawn-tracker.ts | 30 ++++ 6 files changed, 343 insertions(+), 16 deletions(-) create mode 100644 tests/helpers/messages-overflow-child.ts create mode 100644 tests/helpers/messages-overflow-spawn-tracker.ts diff --git a/scripts/ci/test-durations.windows.tsv b/scripts/ci/test-durations.windows.tsv index 8160bf7f196..5ee31b6bd6c 100644 --- a/scripts/ci/test-durations.windows.tsv +++ b/scripts/ci/test-durations.windows.tsv @@ -1,7 +1,7 @@ # Per-file Bun test durations in milliseconds, read from hosted CI job logs. # Consumed by scripts/ci/run-bun-test-batches.sh to balance shards; a file without a row weighs the median. # Regenerate with scripts/ci/test-durations.ts (usage in its header); do not edit by hand. -# Source: runs 41491122166 (workflow_dispatch on dev 1800b4d7f: windows 7/11, 8/11, 10/11 failed and 9/11 cancelled; files measured to completion count) and 41462914814 (dev 09b558580), windows 1/11-11/11; prior rows retained for unmeasured files +# Source: runs 41491122166 (workflow_dispatch on dev 1800b4d7f: windows 7/11, 8/11, 10/11 failed and 9/11 cancelled; files measured to completion count) and 41462914814 (dev 09b558580), windows 1/11-11/11; prior rows retained for unmeasured files; provisional overflow primary groups: 42161735382 jobs 127708835065 baseline pass, 127708835193 builtin 4 timed out, 127708835075 collectors pass, 127708835100 stream pass, 127708835179 verdicts pass; 42168265793 jobs 127730254061 native 1 timed out, 127730254018 verdicts pass; 42145900556 job 127657289210 translated 3 timed out 4873 tests/adapters/abort-race.test.ts 2703 tests/adapters/adapter-buffered-tool-conformance.test.ts 897 tests/adapters/adapter-error-inline.test.ts @@ -427,6 +427,13 @@ 2818 tests/claude-integration/messages-native-opaque-state.test.ts 15045 tests/claude-integration/messages-native-scope.test.ts 52752 tests/claude-integration/messages-native.test.ts +224 tests/claude-integration/messages-overflow-baseline-contract.test.ts +391302 tests/claude-integration/messages-overflow-builtin.test.ts +9197 tests/claude-integration/messages-overflow-collectors.test.ts +290644 tests/claude-integration/messages-overflow-native.test.ts +8748 tests/claude-integration/messages-overflow-stream.test.ts +346067 tests/claude-integration/messages-overflow-translated.test.ts +255760 tests/claude-integration/messages-overflow-verdicts.test.ts 71096 tests/claude-integration/messages-request-id-endpoint.test.ts 174 tests/claude-integration/messages-request-id-headers.test.ts 45635 tests/claude-integration/messages-revoked-token.test.ts diff --git a/tests/claude-integration/messages-overflow-baseline-contract.test.ts b/tests/claude-integration/messages-overflow-baseline-contract.test.ts index 64f653c6050..be8d1451b07 100644 --- a/tests/claude-integration/messages-overflow-baseline-contract.test.ts +++ b/tests/claude-integration/messages-overflow-baseline-contract.test.ts @@ -1,7 +1,13 @@ import { expect, test } from "bun:test"; import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { isAbsolute, join, relative } from "node:path"; import { repoPath } from "../helpers/repo-root"; +import { collectOverflowChild, finishOverflowScratch, overflowScratchBase, type OverflowChild } from "../helpers/messages-overflow-child"; +import { TEST_TEMP_OWNER_FILE } from "../../scripts/test-temp"; +import { trackOverflowAsyncSpawns } from "../helpers/messages-overflow-spawn-tracker"; +import { setWindowsOwnerAclAsyncRunnerForTests, verifyWindowsEntriesAclAsync } from "../../src/lib/windows-owner-acl"; const digest = (text: string | Uint8Array) => createHash("sha256").update(text).digest("hex"); test("#8128 shared overflow predicate changes only in the documented 5xx and conditional cases", () => { @@ -22,3 +28,161 @@ test("#8128 behavior suite retains origin/dev source", () => { expect(source.length).toBeGreaterThan(0); expect(digest(source)).toBe("d5d0573f377c5d7894bff343a0bd9cebec46df5bc405b112238862479a3e7328"); }); + +test("retained overflow scratch is outside nested test sandboxes and junction aliases", () => { + const root = mkdtempSync(join(tmpdir(), "ocx-overflow-location-")); + try { + const nested = join(root, "nested"); + const inner = join(nested, "tmp"); + mkdirSync(inner, { recursive: true }); + writeFileSync(join(root, TEST_TEMP_OWNER_FILE), "{}"); + writeFileSync(join(nested, TEST_TEMP_OWNER_FILE), "{}"); + const alias = join(root, "alias"); + symlinkSync(inner, alias, process.platform === "win32" ? "junction" : "dir"); + const base = overflowScratchBase(alias); + const rel = relative(realpathSync.native(root), base); + expect(rel === ".." || rel.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) || isAbsolute(rel)).toBe(true); + expect(base).toBe(overflowScratchBase(inner)); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("overflow child drains both streams before returning its exit", async () => { + const child = Bun.spawn([process.execPath, "-e", "process.stdout.write('out'); process.stderr.write('err');"], { stdout: "pipe", stderr: "pipe" }); + const result = await collectOverflowChild(child, performance.now() + 25_000); + expect(result).toEqual({ exit: 0, stdout: "out", stderr: "err" }); + expect(child.exitCode).toBe(0); +}, 30_000); + +test("overflow deadline kills and joins the owned real child", async () => { + const child = Bun.spawn([process.execPath, "-e", "process.stdout.write('out'); process.stderr.write('err'); setInterval(() => {}, 1000);"], { stdout: "pipe", stderr: "pipe" }); + let joined = false; + try { + await expect(collectOverflowChild(child, performance.now() + 500)).rejects.toThrow("internal deadline"); + const exit = await child.exited; + joined = true; + expect(exit).not.toBe(0); + } finally { + if (!joined) { try { child.kill(); } finally { await child.exited; } } + } +}, 30_000); + +test("a pipe failure kills and joins the still-live owned child", async () => { + let code: number | null = null; + let resolveExit!: (value: number) => void; + let kills = 0; + const child: OverflowChild = { + pid: 42, + get exitCode() { return code; }, + exited: new Promise(resolve => { resolveExit = resolve; }), + stdout: new ReadableStream({ start(controller) { controller.error(new Error("fixture pipe failure")); } }), + stderr: new ReadableStream({ start(controller) { controller.close(); } }), + kill() { kills++; code = 143; resolveExit(143); }, + }; + await expect(collectOverflowChild(child, performance.now() + 2_000)).rejects.toThrow("cleanup incomplete"); + expect(kills).toBe(1); + expect(await child.exited).toBe(143); +}); + +test("expired deadline is not accepted when a delayed timer loses the race", async () => { + const closed = () => new ReadableStream({ start(controller) { controller.close(); } }); + const child: OverflowChild = { pid: 42, exitCode: 0, exited: Promise.resolve(0), stdout: closed(), stderr: closed(), kill() { throw new Error("already exited"); } }; + await expect(collectOverflowChild(child, performance.now() - 1)).rejects.toThrow("internal deadline"); +}); + +test("unconfirmed termination and a held pipe are reported without waiting forever", async () => { + let kills = 0; + const child: OverflowChild = { + pid: 42, exitCode: null, exited: new Promise(() => {}), + stdout: new ReadableStream({ start(controller) { controller.close(); } }), + stderr: new ReadableStream({}), kill() { kills++; }, + }; + await expect(collectOverflowChild(child, performance.now() + 10, 30)).rejects.toThrow("cleanup incomplete"); + expect(kills).toBe(1); +}); + +test("overflow child preserves a pipe failure and its cleanup failures", async () => { + const child: OverflowChild = { + pid: 42, exitCode: null, exited: new Promise(() => {}), + stdout: new ReadableStream({ start(controller) { controller.error(new Error("fixture pipe failure")); } }), + stderr: new ReadableStream({}), kill() { throw new Error("fixture kill failure"); }, + }; + let caught: unknown; + try { await collectOverflowChild(child, performance.now() + 1_000, 30); } catch (error) { caught = error; } + expect(caught).toBeInstanceOf(AggregateError); + const details = (caught as AggregateError).errors.map(String).join("\n"); + expect(details).toContain("fixture pipe failure"); + expect(details).toContain("fixture kill failure"); + expect(details).toContain("exit or output drain was not confirmed"); +}); + +test("overflow cleanup retains unverified scratch and preserves a primary failure", () => { + const primary = new Error("primary fixture failure"); + let removed = false; + expect(() => finishOverflowScratch("fixture-scratch", false, { error: primary }, () => { removed = true; })).toThrow("primary fixture failure"); + expect(removed).toBe(false); +}); + +test("overflow cleanup reports both the primary and removal failures", () => { + const primary = new Error("primary fixture failure"); + const cleanup = new Error("fixture removal failure"); + let caught: unknown; + try { finishOverflowScratch("fixture-scratch", true, { error: primary }, () => { throw cleanup; }); } catch (error) { caught = error; } + expect(caught).toBeInstanceOf(AggregateError); + expect((caught as AggregateError).errors).toEqual([primary, cleanup]); +}); + +test("overflow subprocess tracker waits for a real child after its caller returns", async () => { + const tracker = trackOverflowAsyncSpawns(); + let child: ReturnType> | undefined; + let exited = false; + let watchdog: ReturnType | undefined; + try { + child = Bun.spawn([process.execPath, "-e", "await Bun.stdin.text();"], { stdin: "pipe", stdout: "pipe", stderr: "pipe" }); + const owned = child; + watchdog = setTimeout(() => { if (!exited) owned.kill(); }, 5_000); + let drained = false; + const draining = tracker.drain().then(() => { drained = true; }); + await Promise.resolve(); + expect(drained).toBe(false); + owned.stdin.end(); + await draining; + const exit = await owned.exited; + exited = true; + expect(exit).toBe(0); + expect(drained).toBe(true); + } finally { + clearTimeout(watchdog); + if (child && !exited) { try { child.kill(); } catch { /* already exited */ } await child.exited; } + tracker.restore(); + } +}, 30_000); + +test("overflow tracker joins an ACL verifier child after the verifier reports timeout", async () => { + const tracker = trackOverflowAsyncSpawns(); + let child: ReturnType> | undefined; + let joined = false; + let watchdog: ReturnType | undefined; + setWindowsOwnerAclAsyncRunnerForTests(async () => { + child = Bun.spawn([process.execPath, "-e", "await Bun.stdin.text();"], { stdin: "pipe", stdout: "pipe", stderr: "pipe" }); + const owned = child; + watchdog = setTimeout(() => { if (!joined) owned.kill(); }, 5_000); + return { success: false, timedOut: true, stdout: new Uint8Array() }; + }); + try { + expect(await verifyWindowsEntriesAclAsync([{ path: "fixture", directory: true }], 1_000)).toBe(false); + if (!child) throw new Error("ACL fixture did not spawn its child"); + let drained = false; + const draining = tracker.drain().then(() => { drained = true; }); + await Promise.resolve(); + expect(drained).toBe(false); + child.stdin.end(); + await draining; + expect(await child.exited).toBe(0); + joined = true; + } finally { + clearTimeout(watchdog); + setWindowsOwnerAclAsyncRunnerForTests(null); + if (child && !joined) { try { child.kill(); } catch { /* already exited */ } await child.exited; } + tracker.restore(); + } +}, 30_000); diff --git a/tests/helpers/messages-overflow-assertions.ts b/tests/helpers/messages-overflow-assertions.ts index 4cdd06856b3..d666856a8e2 100644 --- a/tests/helpers/messages-overflow-assertions.ts +++ b/tests/helpers/messages-overflow-assertions.ts @@ -1,11 +1,11 @@ import { messagesFold502Cases } from "./messages-sse-fold-exceptions"; import { hostedWindowsQuarantine } from "./windows-ci-quarantine"; import { expect, test } from "bun:test"; -import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { mkdtempSync, mkdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { ANTHROPIC_ERROR_TYPES, ANTHROPIC_ERROR_CODES } from "../../src/lib/client-error-body"; import { fixturePath, helperPath, repoRoot } from "./repo-root"; +import { collectOverflowChild, finishOverflowScratch, overflowScratchBase } from "./messages-overflow-child"; import { applicableMessagesOverflowFixtures, messagesOverflowLanes, messagesOverflowCaseId, messagesOverflowCaseTableHash, type MessagesOverflowFixture } from "./messages-overflow-cases"; type Observation = { status: number; bodySha256: string; bodyLength: number; clientError: boolean; envelopeType?: string; mediaType: string | null; type?: string; code?: string | null; text: string; clientRecognizes: boolean; compactionSentence: boolean; internalDecisions: Record }; @@ -22,6 +22,9 @@ if (golden.caseTableSha256 !== messagesOverflowCaseTableHash()) { } export const cases = messagesOverflowLanes.flatMap(lane => applicableMessagesOverflowFixtures(lane).map(input => ({ lane, input }))); export const key = (row: Pick) => messagesOverflowCaseId(row.lane, row.input); +// Hosted Windows 64-row subprocesses for these network-backed lanes approached +// or exceeded 30s; the pure converter/collector lanes finished in under 1s. +const WINDOWS_SLOW_LANES = new Set(["passthrough", "count_tokens", "native-http", "native-fold", "native-fold-builtin", "translated-http", "translated-fold", "translated-json"]); // Reviewed #8148 deltas against the immutable #8153 baseline. No runtime classifier // chooses expectations: only these five fixtures on the two direct-writer paths change. @@ -122,7 +125,7 @@ function assertRow(base: Pick, current: Row) { expect(after.code).toBe(typeof expected.code === "string" && expected.code !== "thread_unsupported_request" && ANTHROPIC_ERROR_CODES.has(expected.code) ? expected.code : undefined); } -/** Each test owns only one 64-row child run; no shared hook computes the full matrix. */ +/** Each test owns one bounded child run; no shared hook computes the full matrix. */ export function registerMessagesOverflowRows(lanes: readonly Row["lane"][]) { expect(golden.schemaVersion).toBe(8); expect(golden.columns).toEqual(["lane", "status", "clientRecognizes", "decisions", "class", "type", "code", "mediaType", "clientError", "bodySha256?", "bodyLength?"]); @@ -132,23 +135,32 @@ export function registerMessagesOverflowRows(lanes: readonly Row["lane"][]) { expect(golden.baseCommit).toBe("72e80d280d614bdf4f1aada9b233f6732404afec"); for (const lane of lanes) { const rows = cases.filter(row => row.lane === lane); - const networkBacked = ["passthrough", "count_tokens", "native-http", "native-fold", "native-fold-builtin", "translated-http", "translated-fold", "translated-json"].includes(lane); - for (let offset = 0; offset < rows.length; offset += 64) { - const selected = rows.slice(offset, offset + 64); - test.skipIf(hostedWindowsQuarantine && networkBacked)(`${lane}: origin/dev rows ${offset + 1}-${offset + selected.length}`, async () => { + const sliceSize = process.platform === "win32" && WINDOWS_SLOW_LANES.has(lane) ? 16 : 64; + const slices = Array.from({ length: Math.ceil(rows.length / sliceSize) }, (_, index) => rows.slice(index * sliceSize, (index + 1) * sliceSize)); + expect(slices.flat().map(key)).toEqual(rows.map(key)); + for (const [index, selected] of slices.entries()) { + const offset = index * sliceSize; + // #8262 hosted-Windows quarantine stays until a Windows lane=all run passes with it removed. + test.skipIf(hostedWindowsQuarantine && WINDOWS_SLOW_LANES.has(lane))(`${lane}: origin/dev rows ${offset + 1}-${offset + selected.length}`, async () => { const started = performance.now(); - const scratch = mkdtempSync(join(tmpdir(), "ocx-overflow-slice-")); + const scratch = mkdtempSync(join(overflowScratchBase(), "ocx-overflow-slice-")); + let safeToRemove = true; + let failure: { error: unknown } | undefined; try { - mkdirSync(join(scratch, "ocx")); mkdirSync(join(scratch, "codex")); + mkdirSync(join(scratch, "ocx")); mkdirSync(join(scratch, "codex")); mkdirSync(join(scratch, "tmp")); const output = join(scratch, "head.json"); - const child = Bun.spawn([process.execPath, helperPath("messages-overflow-differential.ts"), "head", scratch, output, lane, String(offset), "64"], { cwd: repoRoot(), env: { ...process.env, OPENCODEX_HOME: join(scratch, "ocx"), CODEX_HOME: join(scratch, "codex") }, stdout: "pipe", stderr: "pipe" }); - const [exit, stdout, stderr] = await Promise.all([child.exited, new Response(child.stdout).text(), new Response(child.stderr).text()]); + const child = Bun.spawn([process.execPath, helperPath("messages-overflow-differential.ts"), "head", scratch, output, lane, String(offset), String(sliceSize)], { cwd: repoRoot(), env: { ...process.env, HOME: scratch, USERPROFILE: scratch, TEMP: join(scratch, "tmp"), TMP: join(scratch, "tmp"), TMPDIR: join(scratch, "tmp"), OPENCODEX_HOME: join(scratch, "ocx"), CODEX_HOME: join(scratch, "codex"), CODEX_SQLITE_HOME: join(scratch, "codex"), OCX_OWNER_REGISTRY_DIR: join(scratch, "ocx", "ocx-homes"), CLAUDE_CONFIG_DIR: join(scratch, "claude"), GROK_HOME: join(scratch, "grok") }, stdout: "pipe", stderr: "pipe" }); + safeToRemove = false; + const { exit, stdout, stderr } = await collectOverflowChild(child, process.platform === "win32" ? started + 25_000 : null); if (exit !== 0) throw new Error(`Head slice exited ${exit}\n${stdout}\n${stderr}`); - const result = JSON.parse(readFileSync(output, "utf8")) as { rows: Row[] }; + const result = JSON.parse(readFileSync(output, "utf8")) as { rows: Row[]; fixtureAsyncSpawnsDrained?: boolean }; + if (result.fixtureAsyncSpawnsDrained !== true) throw new Error("Head slice did not confirm fixture subprocess cleanup"); + safeToRemove = true; expect(result.rows).toHaveLength(selected.length); for (const [index, row] of result.rows.entries()) { expect(key(row)).toBe(key(selected[index]!)); assertRow(selected[index]!, row); } console.info(`MESSAGES_OVERFLOW_ROWS ${lane} ${offset} ${selected.length} ${Math.round(performance.now() - started)}ms`); - } finally { rmSync(scratch, { recursive: true, force: true }); } + } catch (error) { failure = { error }; } + finally { finishOverflowScratch(scratch, safeToRemove, failure); } }, 30000); } } diff --git a/tests/helpers/messages-overflow-child.ts b/tests/helpers/messages-overflow-child.ts new file mode 100644 index 00000000000..84ad3d57dbb --- /dev/null +++ b/tests/helpers/messages-overflow-child.ts @@ -0,0 +1,98 @@ +import { lstatSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { TEST_TEMP_OWNER_FILE } from "../../scripts/test-temp"; + +/** Place retained child state outside every parent test-sandbox cleanup root. */ +export function overflowScratchBase(temp = tmpdir()): string { + const canonical = realpathSync.native(temp); + let current = canonical; + let outermost: string | undefined; + for (;;) { + try { lstatSync(join(current, TEST_TEMP_OWNER_FILE)); outermost = current; } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + const parent = dirname(current); + if (parent === current) break; + current = parent; + } + if (!outermost) return canonical; + const outside = dirname(outermost); + if (outside === outermost) throw new Error("test sandbox cannot be escaped safely"); + return outside; +} + +/** Preserve a test failure if scratch cleanup also fails; never remove unverified state. */ +export function finishOverflowScratch( + root: string, + safeToRemove: boolean, + failure: { error: unknown } | undefined, + remove: (path: string) => void = path => rmSync(path, { recursive: true, force: true }), +): void { + let cleanupError: unknown; + let cleanupFailed = false; + try { + if (safeToRemove) remove(root); + else console.error(`Overflow child cleanup was not confirmed; retained scratch: ${root}`); + } catch (error) { cleanupError = error; cleanupFailed = true; } + if (failure && cleanupFailed) throw new AggregateError([failure.error, cleanupError], "overflow failure and scratch cleanup failure"); + if (failure) throw failure.error; + if (cleanupFailed) throw cleanupError; +} + +/** The overflow test owns this direct child and both of its output streams. */ +export interface OverflowChild { + readonly pid: number; + readonly exitCode: number | null; + readonly exited: Promise; + readonly stdout: ReadableStream; + readonly stderr: ReadableStream; + kill(): unknown; +} + +type Outcome = { exit: number; stdout: string; stderr: string }; + +/** + * An internal deadline leaves time inside Bun's unchanged outer test budget to + * stop and join the direct child. An unsuccessful call never authorizes the + * caller to remove its scratch: a terminated child may have descendants. + */ +export async function collectOverflowChild(child: OverflowChild, deadline: number | null, reapMs = 2_000): Promise { + let exitObserved = false; + const exit = Promise.resolve().then(() => child.exited).then(code => { exitObserved = true; return code; }); + const stdout = Promise.resolve().then(() => new Response(child.stdout).text()); + const stderr = Promise.resolve().then(() => new Response(child.stderr).text()); + // Attach handlers immediately, including to the losing completion branch. + const joined = Promise.allSettled([exit, stdout, stderr]); + const completion = Promise.all([exit, stdout, stderr]).then( + ([code, out, err]) => ({ kind: "complete" as const, value: { exit: code, stdout: out, stderr: err } }), + error => ({ kind: "failed" as const, error }), + ); + let timer: ReturnType | undefined; + const timeout = new Promise<{ kind: "timeout" }>(resolve => { + if (deadline !== null) timer = setTimeout(() => resolve({ kind: "timeout" }), Math.max(0, deadline - performance.now())); + }); + try { + const first = await Promise.race([completion, timeout]); + if (first.kind === "complete" && (deadline === null || performance.now() < deadline)) return first.value; + const primary = first.kind === "failed" ? first.error : new Error("overflow child exceeded its internal deadline"); + const cleanupErrors: unknown[] = []; + // On Windows Bun can resolve `exited` with 143 while leaving exitCode null. + // The joined exit promise, not that stale property, owns the lifetime. + if (!exitObserved) { + try { child.kill(); } catch (error) { if (!exitObserved) cleanupErrors.push(error); } + } + let reapTimer: ReturnType | undefined; + const remaining = deadline === null ? reapMs : Math.max(0, Math.min(reapMs, deadline + reapMs - performance.now())); + const cleanup = await Promise.race([ + joined, + new Promise(resolve => { reapTimer = setTimeout(() => resolve(null), remaining); }), + ]); + clearTimeout(reapTimer); + if (cleanup === null) cleanupErrors.push(new Error(`child ${child.pid} exit or output drain was not confirmed`)); + else for (const result of cleanup) if (result.status === "rejected") cleanupErrors.push(result.reason); + if (cleanupErrors.length) throw new AggregateError([primary, ...cleanupErrors], `overflow child ${child.pid} failed; cleanup incomplete`); + throw primary; + } finally { + clearTimeout(timer); + } +} diff --git a/tests/helpers/messages-overflow-differential.ts b/tests/helpers/messages-overflow-differential.ts index 5290c4d98eb..b424d2b0780 100644 --- a/tests/helpers/messages-overflow-differential.ts +++ b/tests/helpers/messages-overflow-differential.ts @@ -65,6 +65,9 @@ function replaceOnce(source: string, needle: string, replacement: string): strin build.onLoad({ filter }, ({ path }) => ({ contents: readFileSync(overlays.get(path)!, "utf8"), loader: "ts" })); } }); } +// Install before the request graph loads, so even an ACL verifier that returns +// after requesting termination cannot hide its still-live PowerShell child. +const spawnTracker = base ? undefined : (await import("./messages-overflow-spawn-tracker")).trackOverflowAsyncSpawns(); const { runMessagesOverflowCase } = await import("./messages-overflow-case"); const { upstreamMessagesRequestIdHeaders } = await import("../../src/server/messages-response-headers"); if (base && upstreamMessagesRequestIdHeaders(new Headers({ "request-id": "req_fixture" }))["request-id"] !== "req_fixture") throw new Error("Baseline modules were not loaded"); @@ -118,4 +121,17 @@ if (base) { const groups = []; for (let index = 0; index < lines.length; index += 3) groups.push(lines.slice(index, index + 3).join(",")); await Bun.write(output, `${JSON.stringify(header).slice(0, -1)},"outcomes":{\n${groups.join(",\n")}\n}}\n`); -} else await Bun.write(output, JSON.stringify({ schemaVersion: 8, rows }) + "\n"); +} else { + // Successful exit is not enough on Windows: an async ACL child can outlive its caller. + const { settleBackgroundWork } = await import("../../src/lib/background-work"); + const settled = await settleBackgroundWork({ timeoutMs: 2_000 }); + if (!settled.settled) throw new Error(`Overflow background work did not settle: ${settled.pending.join(", ")}`); + const { flushConfigDirHardeningForTests } = await import("../../src/config/paths"); + const { flushWindowsSecretAclReapsBeforeRemoval, windowsSecretAclReapPendingAtOrBelow } = await import("../../src/lib/windows-secret-acl"); + await flushConfigDirHardeningForTests(); + await flushWindowsSecretAclReapsBeforeRemoval(scratch); + if (windowsSecretAclReapPendingAtOrBelow(scratch)) throw new Error("Overflow ACL child remains active"); + await spawnTracker!.drain(); + spawnTracker!.restore(); + await Bun.write(output, JSON.stringify({ schemaVersion: 8, rows, fixtureAsyncSpawnsDrained: true }) + "\n"); +} diff --git a/tests/helpers/messages-overflow-spawn-tracker.ts b/tests/helpers/messages-overflow-spawn-tracker.ts new file mode 100644 index 00000000000..471d51c91b1 --- /dev/null +++ b/tests/helpers/messages-overflow-spawn-tracker.ts @@ -0,0 +1,30 @@ +/** + * Fixture-only observation of async Bun subprocesses. Some Windows read-only + * ACL probes return on timeout before their PowerShell child has actually exited. + * This wrapper preserves the real spawn and keeps its exit promise until joined. + */ +export function trackOverflowAsyncSpawns() { + const original = Bun.spawn; + const pending = new Set>(); + const failures: unknown[] = []; + const wrapped = ((...args: unknown[]) => { + const child = Reflect.apply(original, Bun, args) as ReturnType; + try { + const observed = Promise.resolve(child.exited).then(() => {}, error => { failures.push(error); }); + pending.add(observed); + void observed.then(() => pending.delete(observed)); + } catch (error) { failures.push(error); } + return child; + }) as typeof Bun.spawn; + Bun.spawn = wrapped; + return { + async drain() { + while (pending.size) await Promise.all([...pending]); + if (failures.length) throw new AggregateError(failures, "overflow subprocess exit was not observed"); + }, + restore() { + if (Bun.spawn !== wrapped) throw new Error("overflow subprocess tracker was replaced"); + Bun.spawn = original; + }, + }; +} From 508a535634755c357b5d5adbcce1799d4ebcf67d Mon Sep 17 00:00:00 2001 From: JUN Date: Sat, 7 Nov 2026 12:50:10 +0900 Subject: [PATCH 04/26] fix(link): hedge leftover-pin refusal and count in-flight readiness reads (#8261) * fix(link): hedge leftover-pin refusal and count in-flight readiness reads * fix(link): keep identity pins that are not links of the token --- docs-site/src/content/docs/guides/remote-link.md | 2 +- src/client/orphan-token-recovery.ts | 2 +- tests/clients/client-machine-listener.test.ts | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs-site/src/content/docs/guides/remote-link.md b/docs-site/src/content/docs/guides/remote-link.md index a162851a342..25aa0d8fd11 100644 --- a/docs-site/src/content/docs/guides/remote-link.md +++ b/docs-site/src/content/docs/guides/remote-link.md @@ -77,7 +77,7 @@ On a Child, `GET /healthz` only says that OpenCodex is running. `GET /readyz` (a The reason `client_state_changed` means the running Child's saved connection is missing or now names another owner or link. Its machine and link status endpoints also return `503` with this reason. Stop or restart the Child and check its connection before re-linking; this readiness result does not itself disconnect the relay. -If `ocx connect status --json` reports `disconnected` but a leftover service token prevents a new link, inspect the situation on the Child. After confirming that preserving the existing token files is intended, run `ocx disconnect --orphaned-token --yes`. It refuses a connected or pending client and any live client runtime or unverified runtime state. It also refuses a Hub, a non-loopback bind, a configured or matching admission secret, and an installed or unverified service: these can use the token for admission. On a Hub, manage the credential through the Hub service; otherwise inspect the bind, API keys, token environment and `ocx service status`, and remove or migrate a service only if intended before retrying. It moves the token and any rotation backup into a private directory under the OpenCodex config directory, prints the location, and does not infer that either token belonged to the link. On Windows, recovery requires hard-link support to preserve file identity while closing handles; if it reports an incomplete backup, inspect the reported directory and original files before retrying. If recovery was interrupted and a retry refuses with `unsafe token file` because another hard link exists, remove the leftover `.service-api-token.identity` (or `.service-api-token.prev.identity`) file from the `orphaned-token-*` folder after checking that the original token file is still in place, then retry. Stop OpenCodex to restore: move the saved files back only when the original names are absent. The command does not revoke a Home key or restore missing client configuration. +If `ocx connect status --json` reports `disconnected` but a leftover service token prevents a new link, inspect the situation on the Child. After confirming that preserving the existing token files is intended, run `ocx disconnect --orphaned-token --yes`. It refuses a connected or pending client and any live client runtime or unverified runtime state. It also refuses a Hub, a non-loopback bind, a configured or matching admission secret, and an installed or unverified service: these can use the token for admission. On a Hub, manage the credential through the Hub service; otherwise inspect the bind, API keys, token environment and `ocx service status`, and remove or migrate a service only if intended before retrying. It moves the token and any rotation backup into a private directory under the OpenCodex config directory, prints the location, and does not infer that either token belonged to the link. On Windows, recovery requires hard-link support to preserve file identity while closing handles; if it reports an incomplete backup, inspect the reported directory and original files before retrying. If recovery was interrupted and a retry refuses with `unsafe token file` because another hard link exists, check the `orphaned-token-*` folders for a leftover `.service-api-token.identity` (or `.service-api-token.prev.identity`) file. Remove it only if it is a hard link of the current token file (the same file, not just the same name); keep any other pin, because it may be the only copy of an older token. Then inspect any remaining hard links to the token file before retrying. Stop OpenCodex to restore: move the saved files back only when the original names are absent. The command does not revoke a Home key or restore missing client configuration. To confirm which Home is serving a Child, compare the `x-opencodex-served-by` header on the Child's `GET /v1/models` with the `instanceId` in the Home's `GET /healthz`. The Child passes the Home's header through unchanged, so the two match while the link works. The value is an opaque id, never a hostname; see [Home identity and the served-by header](/reference/management-api/#home-identity-and-the-served-by-header). diff --git a/src/client/orphan-token-recovery.ts b/src/client/orphan-token-recovery.ts index ca16b3e2ffb..2c506a82cee 100644 --- a/src/client/orphan-token-recovery.ts +++ b/src/client/orphan-token-recovery.ts @@ -149,7 +149,7 @@ function openSource(path: string, name: string): OpenSource | null { catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; } if (entry.isFile() && !entry.isSymbolicLink() && entry.nlink > 1) { // An interrupted Windows recovery leaves its identity pin linked to the original. - throw new Error(`orphan recovery refused: unsafe token file (another hard link exists; if an earlier recovery was interrupted, remove the leftover .${name}.identity pin from its orphaned-token-* backup folder and retry)`); + throw new Error(`orphan recovery refused: unsafe token file (another hard link exists; an interrupted earlier recovery may have left a .${name}.identity pin in an orphaned-token-* backup folder. Remove such a pin only if it is a hard link of this token file and keep any other pin, then inspect any remaining hard links to this file before retrying)`); } if (!entry.isFile() || entry.isSymbolicLink() || entry.nlink !== 1 || entry.size > 4096 || (process.platform !== "win32" && entry.uid !== process.getuid?.())) throw new Error("orphan recovery refused: unsafe token file"); diff --git a/tests/clients/client-machine-listener.test.ts b/tests/clients/client-machine-listener.test.ts index c619653dd0d..1d34b40d96e 100644 --- a/tests/clients/client-machine-listener.test.ts +++ b/tests/clients/client-machine-listener.test.ts @@ -392,6 +392,7 @@ describe("client machine listener in link mode", () => { state, managementAuthState: authState(), linkReadiness: async () => { notifyProbe(); await pending; return { ready: true }; }, }); servers.push(server); + const reads = spyOn(clientState, "readClientConnectionState"); try { const responsePromise = fetch(new URL("/readyz", server.url)); await started; @@ -400,7 +401,8 @@ describe("client machine listener in link mode", () => { const response = await responsePromise; expect(response.status).toBe(503); expect(await response.json()).toMatchObject({ link: { state: "unavailable", reason: "client_state_changed" } }); - } finally { resolveProbe(); } + expect(reads).toHaveBeenCalledTimes(1); + } finally { resolveProbe(); reads.mockRestore(); } }); test("durable state is read once per readiness and never for unrelated machine routes", async () => { From 05eaaa38867be932e9384126acbc4133ab240713 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sat, 7 Nov 2026 12:57:43 +0900 Subject: [PATCH 05/26] test: stabilize Windows fixture setup and source checks (#8176) * test(messages): fix Windows fixture hooks and spend-owner lifetime * test(config): retry busy ownership lock in first-registration race children The inline child used by "processes that start together on an absent home" exited 1 whenever recordOwnedConfigPath threw CONFIG_OWNERSHIP_LOCK_UNAVAILABLE on a contended first-write lock (CI #8176). Match the sibling childScript retry: bounded 400 attempts, 5 ms backoff, only for that error code; other errors still exit immediately. Barrier, 60 s timeout, manifest assertions and the production 50 ms lock budget are unchanged. * docs: identify the tracked inventory required by sparse test checkouts * test: honor conditional skips in Bun ownership fixtures * test: isolate and settle Windows provider fixtures Await tracked cleanup, disable irrelevant native Codex synchronization in synthetic provider configs, and seed account sets through one real store mutation. Keep metrics stale rows first after sorting. Separate Kiro server setup with a bounded 10-second hook while API assertions retain the default 5-second limit. * test: bound source oracle reads without extending deadlines * test: separate fixture preparation and teardown deadlines * test(messages): separate 529 fixture setup and async teardown * test(anthropic): restore fixture environment when shutdown rejects Attempt both server stops and restore both environment roots in finally. Preserve undrained trees and propagate failures. Controlled rejecting-stop regression fails before the change; all five file cases pass with 32 assertions on Windows. * test(anthropic): cover fixture drain failure retention * test(tray): wait for published PID in probe readiness --- AGENTS.md | 8 +- .../anthropic-image-retry-e2e.test.ts | 154 +++++++++++++----- .../claude-529-mapping.test.ts | 150 ++++++++--------- .../messages-error-contracts.test.ts | 38 +++-- tests/helpers/messages-overflow-case.ts | 9 +- .../helpers/messages-overflow-differential.ts | 67 +++++--- tests/helpers/remove-tree.ts | 4 + .../forward-admission-separation.test.ts | 9 +- .../kiro/kiro-pool-load-settings.test.ts | 62 ++++--- .../providers/kiro/kiro-quota-metrics.test.ts | 29 +++- tests/windows/windows-tray.test.ts | 11 +- 11 files changed, 335 insertions(+), 206 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f71dc2dd0f3..d5259be3a24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -113,10 +113,10 @@ different pointer, so every branch move and promotion dragged a diff. Reference *clones* are gitignored: they are third-party source carrying their own licenses and have no business in this repository's history. -Nothing in the build, typecheck, or test path reads from `devlog/`, so a -contributor who ignores it entirely still passes every gate. `privacy:scan` does -read it — that is deliberate, and it is what makes a public devlog safe rather -than merely visible. +Build and typecheck do not read `devlog/`. The test-layout membership oracle reads +`devlog/_fin/260905_test_modularization_and_windows/001_test_inventory.md`; sparse +checkouts running that test must materialize this tracked input. `privacy:scan` +also reads `devlog/` to keep public development notes safe to publish. Two mechanical guards in `tests/ci-workflows/repo-hygiene.test.ts` back this up: no `160000` gitlink may be tracked anywhere, and neither the vendored reference clones nor diff --git a/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts b/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts index da69051e673..4a6518081bf 100644 --- a/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts +++ b/tests/adapters/anthropic/anthropic-image-retry-e2e.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync} from "node:fs"; +import { existsSync, mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { saveConfig } from "../../../src/config"; @@ -8,16 +8,20 @@ import { startServer } from "../../../src/server"; import { resetNormalizeStateForTests } from "../../../src/adapters/anthropic-image-normalize"; import { sniffImageDimensions } from "../../../src/adapters/anthropic-image-guard"; import type { OcxConfig } from "../../../src/types"; +import { trackBackgroundWork } from "../../../src/lib/background-work"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../../helpers/isolated-codex-home"; -import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../../helpers/remove-tree"; let testDir = ""; let previousHome: string | undefined; +let previousCodexHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; let upstream: ReturnType | null = null; +let proxy: ReturnType | undefined; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; isolatedCodexHome = installIsolatedCodexHome("ocx-imgretry-e2e-codex-"); testDir = mkdtempSync(join(tmpdir(), "ocx-imgretry-e2e-")); process.env.OPENCODEX_HOME = testDir; @@ -25,16 +29,88 @@ beforeEach(() => { resetNormalizeStateForTests(); }); -afterEach(() => { - upstream?.stop(true); - upstream = null; - if (previousHome === undefined) delete process.env.OPENCODEX_HOME; - else process.env.OPENCODEX_HOME = previousHome; - isolatedCodexHome?.restore(); +/** Restore process state even when shutdown rejects; retain trees until drain succeeds. */ +async function cleanupFixture(): Promise { + const codexHome = isolatedCodexHome; + const directory = testDir; isolatedCodexHome = null; - if (testDir) removeTreeWithRetry(testDir); - clearKeyCooldowns(); -}); + testDir = ""; + try { + try { await proxy?.stop(true); } + finally { + proxy = undefined; + try { await upstream?.stop(true); } + finally { upstream = null; } + } + if (directory) await removeTreeAfterBackgroundWork(directory); + } finally { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome?.restoreEnvironment(); + clearKeyCooldowns(); + } + // A failed stop or drain must leave both owned trees intact for their wrapper. + codexHome?.removeAfterDrain(); +} + +afterEach(cleanupFixture, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); + +for (const rejecting of ["proxy", "upstream"] as const) { + test(`fixture cleanup restores environment after ${rejecting} shutdown rejects`, async () => { + const directory = testDir, codexHome = isolatedCodexHome!; + const expectedHome = previousHome; + const failure = new Error(`synthetic ${rejecting} stop failure`); + const stopped: string[] = []; + // No real producer starts: these controlled failures exercise the hook itself. + proxy = { async stop() { stopped.push("proxy"); if (rejecting === "proxy") throw failure; } } as unknown as typeof proxy; + upstream = { async stop() { stopped.push("upstream"); if (rejecting === "upstream") throw failure; } } as unknown as typeof upstream; + try { + await expect(cleanupFixture()).rejects.toBe(failure); + expect(stopped).toEqual(["proxy", "upstream"]); + expect(proxy).toBeUndefined(); + expect(upstream).toBeNull(); + expect(isolatedCodexHome).toBeNull(); + expect(process.env.OPENCODEX_HOME).toBe(expectedHome); + expect(process.env.CODEX_HOME).toBe(previousCodexHome); + expect(existsSync(directory)).toBe(true); + expect(existsSync(codexHome.path)).toBe(true); + } finally { + // The synthetic stoppers own no live handles; clean up even if a regression fails. + proxy = undefined; upstream = null; isolatedCodexHome = null; testDir = ""; + if (expectedHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = expectedHome; + codexHome.restoreEnvironment(); + await removeTreeAfterBackgroundWork(directory); + codexHome.removeAfterDrain(); + } + }); +} + +test("fixture cleanup restores environment and retains trees when drain fails", async () => { + const directory = testDir, codexHome = isolatedCodexHome!; + const expectedHome = previousHome; + let release!: () => void; + const work = trackBackgroundWork("image-retry fixture drain rejection", new Promise(resolve => { release = resolve; })); + try { + await expect(cleanupFixture()).rejects.toThrow("in-process background work still running: image-retry fixture drain rejection"); + expect(proxy).toBeUndefined(); + expect(upstream).toBeNull(); + expect(isolatedCodexHome).toBeNull(); + expect(process.env.OPENCODEX_HOME).toBe(expectedHome); + expect(process.env.CODEX_HOME).toBe(previousCodexHome); + expect(existsSync(directory)).toBe(true); + expect(existsSync(codexHome.path)).toBe(true); + } finally { + release(); + await work; + proxy = undefined; upstream = null; isolatedCodexHome = null; testDir = ""; + if (expectedHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = expectedHome; + codexHome.restoreEnvironment(); + await removeTreeAfterBackgroundWork(directory); + codexHome.removeAfterDrain(); + } +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); const ONE_PX_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="; @@ -81,6 +157,7 @@ function anthropicConfig(baseUrl: string, pool = false): OcxConfig { return { port: 0, hostname: "127.0.0.1", + clientIntegrations: { codex: false }, defaultProvider: "anthropic-test", providers: { "anthropic-test": { @@ -121,14 +198,30 @@ async function postImageRequest(serverUrl: string, dataUrl: string): Promise void | Promise, +): void { + describe(name, () => { + let seen: SeenRequest[]; + let dataUrl: string; + // Real Windows config/ACL and server setup is separate from the 5s request assertions. + beforeEach(async () => { + seen = []; + upstream = scriptedUpstream(statuses, seen); + saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""), pool)); + proxy = startServer(0); + dataUrl = await realPngDataUrl(1500, 1000); + }, 10_000); + test("preserves the response and retry contract", async () => { + const res = await postImageRequest(String(proxy!.url), dataUrl); + await verify(res, seen); + }); + }); +} + describe("anthropic 413 tightened-retry (end-to-end)", () => { - test("R1: upstream 413 then 200 — exactly one biased rebuild, images re-encoded a tier lower", async () => { - const seen: SeenRequest[] = []; - upstream = scriptedUpstream([413, 200], seen); - saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""))); - const server = startServer(0); - try { - const res = await postImageRequest(String(server.url), await realPngDataUrl(1500, 1000)); + imageRetryScenario("R1: upstream 413 then 200 — exactly one biased rebuild, images re-encoded a tier lower", [413, 200], false, async (res, seen) => { expect(res.status).toBe(200); expect(seen).toHaveLength(2); // First attempt: 1500px PNG rides tier-0 pass-through. @@ -139,18 +232,9 @@ describe("anthropic 413 tightened-retry (end-to-end)", () => { expect(second?.media_type).toBe("image/jpeg"); const dims = sniffImageDimensions(second?.data ?? ""); expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(1024); - } finally { - await server.stop(true); - } }); - test("R2: upstream 413 twice — exactly two calls, honest 413 surfaces, no spiral", async () => { - const seen: SeenRequest[] = []; - upstream = scriptedUpstream([413, 413], seen); - saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""))); - const server = startServer(0); - try { - const res = await postImageRequest(String(server.url), await realPngDataUrl(1500, 1000)); + imageRetryScenario("R2: upstream 413 twice — exactly two calls, honest 413 surfaces, no spiral", [413, 413], false, async (res, seen) => { expect(res.status).toBe(413); expect(seen).toHaveLength(2); expect(res.headers.get("content-type")).toContain("application/json"); @@ -160,18 +244,9 @@ describe("anthropic 413 tightened-retry (end-to-end)", () => { type: "invalid_request_error", code: "context_length_exceeded", }); - } finally { - await server.stop(true); - } }); - test("R4: 413 → biased retry → 429 → key rotation keeps the tightened tiers", async () => { - const seen: SeenRequest[] = []; - upstream = scriptedUpstream([413, 429, 200], seen); - saveConfig(anthropicConfig(upstream.url.toString().replace(/\/$/, ""), true)); - const server = startServer(0); - try { - const res = await postImageRequest(String(server.url), await realPngDataUrl(1500, 1000)); + imageRetryScenario("R4: 413 → biased retry → 429 → key rotation keeps the tightened tiers", [413, 429, 200], true, async (res, seen) => { expect(res.status).toBe(200); expect(seen).toHaveLength(3); // Rotation happened after the biased 413 retry hit 429. @@ -181,8 +256,5 @@ describe("anthropic 413 tightened-retry (end-to-end)", () => { expect(third?.media_type).toBe("image/jpeg"); const dims = sniffImageDimensions(third?.data ?? ""); expect(Math.max(dims!.width, dims!.height)).toBeLessThanOrEqual(1024); - } finally { - await server.stop(true); - } }); }); diff --git a/tests/claude-integration/claude-529-mapping.test.ts b/tests/claude-integration/claude-529-mapping.test.ts index e96a9b564f2..3cf45b43dba 100644 --- a/tests/claude-integration/claude-529-mapping.test.ts +++ b/tests/claude-integration/claude-529-mapping.test.ts @@ -7,30 +7,53 @@ import { startServer } from "../../src/server"; import { getRequestLogEntries } from "../../src/server/request-log"; import type { OcxConfig } from "../../src/types"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "../helpers/isolated-codex-home"; -import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../helpers/remove-tree"; let testDir = ""; let previousHome: string | undefined; let isolatedCodexHome: IsolatedCodexHome | null = null; +let upstream: ReturnType | undefined; +let proxy: ReturnType | undefined; +let calls = 0; +let upstreamResponse: () => Response; beforeEach(() => { previousHome = process.env.OPENCODEX_HOME; isolatedCodexHome = installIsolatedCodexHome("ocx-claude-529-"); testDir = mkdtempSync(join(tmpdir(), "ocx-claude-529-")); process.env.OPENCODEX_HOME = testDir; -}); + calls = 0; + upstreamResponse = () => Response.json({ error: { message: "bad request shape" } }, { status: 400 }); + upstream = Bun.serve({ port: 0, fetch() { calls++; return upstreamResponse(); } }); + saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); + proxy = startServer(0); +}, 10_000); -afterEach(() => { - if (previousHome === undefined) delete process.env.OPENCODEX_HOME; - else process.env.OPENCODEX_HOME = previousHome; - isolatedCodexHome?.restore(); +afterEach(async () => { + const codexHome = isolatedCodexHome; isolatedCodexHome = null; - if (testDir) removeTreeWithRetry(testDir); -}); + try { + try { await proxy?.stop(true); } + finally { + proxy = undefined; + await upstream?.stop(true); + upstream = undefined; + } + if (testDir) await removeTreeAfterBackgroundWork(testDir); + testDir = ""; + } finally { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + codexHome?.restoreEnvironment(); + } + // A failed drain leaves the owned tree for the wrapper; never delete under its writer. + codexHome?.removeAfterDrain(); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); function nativeConfig(baseUrl: string): OcxConfig { return { port: 0, + clientIntegrations: { codex: false }, defaultProvider: "native", providers: { native: { adapter: "openai-responses", baseUrl, authMode: "forward", allowPrivateNetwork: true }, @@ -47,89 +70,48 @@ function messagesBody(): string { } test("transient upstream 502 -> client 529 overloaded_error; retry fired; log keeps upstream 502", async () => { - let calls = 0; - const upstream = Bun.serve({ - port: 0, - fetch() { - calls++; - return Response.json( - { error: { message: "An error occurred while processing your request. request ID test-42" } }, - // Retry-After: 0 keeps the pre-stream retry backoff instant AND must win over the "2" fallback. - { status: 502, headers: { "Retry-After": "0" } }, - ); - }, + upstreamResponse = () => Response.json( + { error: { message: "An error occurred while processing your request. request ID test-42" } }, + // Retry-After: 0 keeps the pre-stream retry backoff instant AND must win over the "2" fallback. + { status: 502, headers: { "Retry-After": "0" } }, + ); + const response = await fetch(new URL("/v1/messages", proxy!.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: messagesBody(), }); - saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); - const server = startServer(0); - try { - const response = await fetch(new URL("/v1/messages", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: messagesBody(), - }); - // Pre-stream transient retry (010) exhausts its attempts against the persistent 502 … - expect(calls).toBe(3); - // … then the Claude envelope reclassifies the transient 5xx as retryable overload (020). - expect(response.status).toBe(529); - expect(response.headers.get("retry-after")).toBe("0"); - const json = await response.json() as { type?: string; error?: { type?: string; message?: string } }; - expect(json.error?.type).toBe("overloaded_error"); - expect(json.error?.message).toBe("The provider failed to complete the request."); - // Diagnostics stay honest: the request log keeps the upstream status, not the client 529. - const entry = getRequestLogEntries().findLast(e => e.surface === "claude"); - expect(entry?.status).toBe(502); - } finally { - await server.stop(true); - await upstream.stop(true); - } + // Pre-stream transient retry (010) exhausts its attempts against the persistent 502 … + expect(calls).toBe(3); + // … then the Claude envelope reclassifies the transient 5xx as retryable overload (020). + expect(response.status).toBe(529); + expect(response.headers.get("retry-after")).toBe("0"); + const json = await response.json() as { type?: string; error?: { type?: string; message?: string } }; + expect(json.error?.type).toBe("overloaded_error"); + expect(json.error?.message).toBe("The provider failed to complete the request."); + // Diagnostics stay honest: the request log keeps the upstream status, not the client 529. + const entry = getRequestLogEntries().findLast(e => e.surface === "claude"); + expect(entry?.status).toBe(502); }, 15_000); test("transient 502 without Retry-After -> fallback Retry-After 2 on the 529 response", async () => { - const upstream = Bun.serve({ - port: 0, - fetch() { - return Response.json({ error: { message: "boom" } }, { status: 502 }); - }, + upstreamResponse = () => Response.json({ error: { message: "boom" } }, { status: 502 }); + const response = await fetch(new URL("/v1/messages", proxy!.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: messagesBody(), }); - saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); - const server = startServer(0); - try { - const response = await fetch(new URL("/v1/messages", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: messagesBody(), - }); - expect(response.status).toBe(529); - expect(response.headers.get("retry-after")).toBe("2"); - } finally { - await server.stop(true); - await upstream.stop(true); - } + expect(response.status).toBe(529); + expect(response.headers.get("retry-after")).toBe("2"); }, 15_000); test("non-transient upstream 400 stays 400 invalid_request_error; no retry", async () => { - let calls = 0; - const upstream = Bun.serve({ - port: 0, - fetch() { - calls++; - return Response.json({ error: { message: "bad request shape" } }, { status: 400 }); - }, + const response = await fetch(new URL("/v1/messages", proxy!.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: messagesBody(), }); - saveConfig(nativeConfig(`${upstream.url.toString().replace(/\/$/, "")}/v1`)); - const server = startServer(0); - try { - const response = await fetch(new URL("/v1/messages", server.url), { - method: "POST", - headers: { "content-type": "application/json" }, - body: messagesBody(), - }); - expect(calls).toBe(1); - expect(response.status).toBe(400); - const json = await response.json() as { error?: { type?: string } }; - expect(json.error?.type).toBe("invalid_request_error"); - } finally { - await server.stop(true); - await upstream.stop(true); - } + expect(calls).toBe(1); + expect(response.status).toBe(400); + const json = await response.json() as { error?: { type?: string } }; + expect(json.error?.type).toBe("invalid_request_error"); }); diff --git a/tests/claude-integration/messages-error-contracts.test.ts b/tests/claude-integration/messages-error-contracts.test.ts index 2460c947686..d5ddeb82e27 100644 --- a/tests/claude-integration/messages-error-contracts.test.ts +++ b/tests/claude-integration/messages-error-contracts.test.ts @@ -134,6 +134,28 @@ for (const [size, cap, type, code] of [[1000, 250, "request_too_large", "transla expect(error.type).toBe(type); expect(error.code).toBe(code); }); +type SourceUses = { path: string; constructors: boolean[]; writers: number }; +let sourceUses: Promise | undefined; +/** Scan every source once with bounded reads; retain only oracle facts, not all source text. */ +function readSourceUses(): Promise { + return sourceUses ??= (async () => { + const paths = [...new Bun.Glob("src/**/*.ts").scanSync(repoRoot())]; + const uses = new Array(paths.length); + let next = 0; + await Promise.all(Array.from({ length: Math.min(8, paths.length) }, async () => { + for (;;) { + const index = next++; + if (index >= paths.length) return; + const path = paths[index]!, source = await Bun.file(repoPath(path)).text(); + uses[index] = { path: path.replaceAll("\\", "/"), + constructors: [...source.matchAll(/new AnthropicRequestError\(/g)].map(call => source.slice(0, call.index).endsWith("markProxyError(")), + writers: source.split("\n").filter(line => /anthropicError(Response|Body)\(/.test(line)).length }; + } + })); + return uses.filter(row => row.constructors.length || row.writers); + })(); +} + const ingressProducerRows: [string, number, string[]][] = [ ["src/claude/inbound.ts", 10, ["File-backed images require native Anthropic passthrough", "tool_result requires tool_use_id", "tool_use requires id and name", "malformed ocxr1 reasoning signature", "OpenCodex reasoning continuity cannot be replayed as an Anthropic signature", "each message must be an object", "unsupported message role: unsupported", "request body must be a JSON object", "model is required", "messages must be a non-empty array"]], ["src/protocols/codecs/messages-ir.ts", 9, ["tool_result requires tool_use_id", "tool_use requires id and name", "MALFORMED_REASONING_SIGNATURE", "OpenCodex reasoning continuity cannot be replayed as an Anthropic signature", "each message must be an object", "unsupported message role: unsupported", "request body must be a JSON object", "model is required", "messages must be a non-empty array"]], @@ -144,11 +166,9 @@ const ingressProducerRows: [string, number, string[]][] = [ ]; test("C1/C2 ingress constructors have explicit producer rows and local identity", async () => { const actual: Record = {}; - for (const path of new Bun.Glob("src/**/*.ts").scanSync(repoRoot())) { - const source = readFileSync(repoPath(path), "utf8"); - const calls = [...source.matchAll(/new AnthropicRequestError\(/g)]; - if (calls.length) actual[path.replaceAll("\\", "/")] = calls.length; - for (const call of calls) expect(source.slice(0, call.index).endsWith("markProxyError(")).toBe(true); + for (const { path, constructors } of await readSourceUses()) { + if (constructors.length) actual[path] = constructors.length; + for (const marked of constructors) expect(marked).toBe(true); } expect(actual).toEqual(Object.fromEntries(ingressProducerRows.map(([path, count]) => [path, count]))); for (const [path, , diagnostics] of ingressProducerRows) { @@ -168,12 +188,10 @@ const envelopeWriterRows = [ ["src/server/index/serve-options.ts", 4, "fixed endpoint auth/origin bypasses"], ["src/server/messages-native-tool-guard.ts", 1, "deferred local guard terminal"], ] as const; -test("W1/W3 envelope writers are covered by explicit dispositions", () => { +test("W1/W3 envelope writers are covered by explicit dispositions", async () => { const actual: Record = {}; - for (const path of new Bun.Glob("src/**/*.ts").scanSync(repoRoot())) { - const lines = readFileSync(repoPath(path), "utf8").split("\n"); - const count = lines.filter(line => /anthropicError(Response|Body)\(/.test(line)).length; - if (count) actual[path.replaceAll("\\", "/")] = count; + for (const { path, writers } of await readSourceUses()) { + if (writers) actual[path] = writers; } expect(actual).toEqual(Object.fromEntries(envelopeWriterRows.map(([path, count]) => [path, count]))); const bypassRows = [ diff --git a/tests/helpers/messages-overflow-case.ts b/tests/helpers/messages-overflow-case.ts index 4327b68fd7c..e19b55a3a24 100644 --- a/tests/helpers/messages-overflow-case.ts +++ b/tests/helpers/messages-overflow-case.ts @@ -49,7 +49,12 @@ export async function observeMessagesOverflowBody(response: Response, lane: Mess return { text, bodySha256, bodyLength }; } -export async function runMessagesOverflowCase(lane: MessagesOverflowLane, fixture: MessagesOverflowFixture) { +/** A slice may lend its real spend owner; single-case callers retain their own lease. */ +export async function runMessagesOverflowCase( + lane: MessagesOverflowLane, + fixture: MessagesOverflowFixture, + acquireSpendHome: () => () => void = acquireOwnedSpendHome, +) { const tail = (fixture.messageStop || fixture.id.startsWith("frame-complete-")) ? 'event: message_stop\ndata: {"type":"message_stop"}\n\n' : ""; const prefix = fixture.prefix ? `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: fixture.prefix } })}\n\nevent: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n` : ""; const error = { ...(fixture.type ? { type: fixture.type } : {}), ...(fixture.status !== undefined ? { status: fixture.status } : {}), message: `${fixture.message} ${overflowSentinel}`, ...(fixture.code !== undefined ? { code: fixture.code } : {}) }; @@ -112,7 +117,7 @@ export async function runMessagesOverflowCase(lane: MessagesOverflowLane, fixtur } else { const passthrough = lane === "passthrough" || lane === "count_tokens"; const native = lane.startsWith("native"); - release = acquireOwnedSpendHome(); clearKeyCooldowns(); + release = acquireSpendHome(); clearKeyCooldowns(); const builtin = lane === "native-fold-builtin"; const nativeFold = lane === "native-fold" || builtin; const reply = () => { diff --git a/tests/helpers/messages-overflow-differential.ts b/tests/helpers/messages-overflow-differential.ts index b424d2b0780..94ba3473029 100644 --- a/tests/helpers/messages-overflow-differential.ts +++ b/tests/helpers/messages-overflow-differential.ts @@ -69,6 +69,7 @@ function replaceOnce(source: string, needle: string, replacement: string): strin // after requesting termination cannot hide its still-live PowerShell child. const spawnTracker = base ? undefined : (await import("./messages-overflow-spawn-tracker")).trackOverflowAsyncSpawns(); const { runMessagesOverflowCase } = await import("./messages-overflow-case"); +const { acquireOwnedSpendHome } = await import("./owned-spend-home"); const { upstreamMessagesRequestIdHeaders } = await import("../../src/server/messages-response-headers"); if (base && upstreamMessagesRequestIdHeaders(new Headers({ "request-id": "req_fixture" }))["request-id"] !== "req_fixture") throw new Error("Baseline modules were not loaded"); const rows = []; @@ -83,35 +84,47 @@ function intern(values: T[], value: T): number { if (index >= 0) return index; return values.push(value) - 1; } -for (const lane of messagesOverflowLanes.filter(lane => !selectedLane || lane === selectedLane)) for (const input of applicableMessagesOverflowFixtures(lane).slice(first, first + count)) { - const observations = new Map(); - (globalThis as any).__messagesOverflowRecord = (tag: string, value: boolean) => { - const values = observations.get(tag) ?? []; values.push(value); observations.set(tag, values); - return value; - }; - if (input.fault) process.stderr.write(`fixture ${lane}/${input.id}\n`); - const result = await runMessagesOverflowCase(lane, input); - const passthrough = lane === "passthrough" || lane === "count_tokens"; - if (!passthrough && input.rawBody === undefined && !input.fault && !input.localBranch && observations.size === 0) throw new Error(`Missing baseline observation for ${lane}/${input.id}`); - const internalDecisions: Record = {}; - for (const [tag, values] of observations) { - if (values.some(value => value !== values[0])) throw new Error(`Inconsistent ${tag} decisions for ${lane}/${input.id}`); - internalDecisions[tag] = values[0]!; - } - if (base) { - const id = messagesOverflowCaseId(lane, input); - if (Object.hasOwn(outcomes, id)) throw new Error(`Duplicate case id: ${lane}/${input.id}`); - // Outcome-only tuples keep the base fixture small; dictionaries retain exact compared values. - const outcome: unknown[] = [messagesOverflowLanes.indexOf(lane), result.status, result.clientRecognizes, - intern(decisions, internalDecisions), input.classification[lane], intern(types, result.type ?? null), - intern(codes, result.code ?? null), intern(mediaTypes, result.mediaType), result.clientError]; - if (input.classification[lane] === "success" || input.fixedLocal) { - outcome.push(result.bodySha256, result.bodyLength); +// A real server owns its spend home for its lifetime. Keep that same ownership +// across this sequential slice, avoiding a full journal replay for every row. +// Pure encoder lanes never acquire a lease; single-case helpers keep their default. +let releaseSpendHome: (() => void) | undefined; +const acquireSliceSpendHome = () => { + releaseSpendHome ??= acquireOwnedSpendHome(); + return () => {}; // Borrowed by the row; the slice's finally owns release. +}; +try { + for (const lane of messagesOverflowLanes.filter(lane => !selectedLane || lane === selectedLane)) for (const input of applicableMessagesOverflowFixtures(lane).slice(first, first + count)) { + const observations = new Map(); + (globalThis as any).__messagesOverflowRecord = (tag: string, value: boolean) => { + const values = observations.get(tag) ?? []; values.push(value); observations.set(tag, values); + return value; + }; + if (input.fault) process.stderr.write(`fixture ${lane}/${input.id}\n`); + const result = await runMessagesOverflowCase(lane, input, acquireSliceSpendHome); + const passthrough = lane === "passthrough" || lane === "count_tokens"; + if (!passthrough && input.rawBody === undefined && !input.fault && !input.localBranch && observations.size === 0) throw new Error(`Missing baseline observation for ${lane}/${input.id}`); + const internalDecisions: Record = {}; + for (const [tag, values] of observations) { + if (values.some(value => value !== values[0])) throw new Error(`Inconsistent ${tag} decisions for ${lane}/${input.id}`); + internalDecisions[tag] = values[0]!; } - outcomes[id] = outcome; - } else rows.push({ lane, input, head: { ...result, internalDecisions } }); + if (base) { + const id = messagesOverflowCaseId(lane, input); + if (Object.hasOwn(outcomes, id)) throw new Error(`Duplicate case id: ${lane}/${input.id}`); + // Outcome-only tuples keep the base fixture small; dictionaries retain exact compared values. + const outcome: unknown[] = [messagesOverflowLanes.indexOf(lane), result.status, result.clientRecognizes, + intern(decisions, internalDecisions), input.classification[lane], intern(types, result.type ?? null), + intern(codes, result.code ?? null), intern(mediaTypes, result.mediaType), result.clientError]; + if (input.classification[lane] === "success" || input.fixedLocal) { + outcome.push(result.bodySha256, result.bodyLength); + } + outcomes[id] = outcome; + } else rows.push({ lane, input, head: { ...result, internalDecisions } }); + } +} finally { + delete (globalThis as any).__messagesOverflowRecord; + releaseSpendHome?.(); } -delete (globalThis as any).__messagesOverflowRecord; if (base) { const header = { schemaVersion: 8, baseCommit: "72e80d280d614bdf4f1aada9b233f6732404afec", caseTableSha256: messagesOverflowCaseTableHash(), columns: ["lane", "status", "clientRecognizes", "decisions", "class", "type", "code", "mediaType", "clientError", "bodySha256?", "bodyLength?"], diff --git a/tests/helpers/remove-tree.ts b/tests/helpers/remove-tree.ts index 5dbf5f2e622..be5079c4532 100644 --- a/tests/helpers/remove-tree.ts +++ b/tests/helpers/remove-tree.ts @@ -3,6 +3,10 @@ import { pendingBackgroundWork, settleBackgroundWork } from "../../src/lib/backg import { flushWindowsSecretAclReapsBeforeRemoval } from "../../src/lib/windows-secret-acl"; import { REMOVE_RETRY_BUDGET_MS, removeTestTempTree, removeTestTempTreeAsync } from "../../scripts/test-temp"; +// Background settlement and removal retries each own one budget. Leave scheduling headroom; +// an unreaped ACL child still blocks removal and may fail this caller-facing deadline. +export const BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS = 2 * REMOVE_RETRY_BUDGET_MS + 5_000; + type RemoveTreeWithRetryOptions = Readonly<{ remove?: (path: string) => void; sleep?: (milliseconds: number) => void; diff --git a/tests/providers/forward-admission-separation.test.ts b/tests/providers/forward-admission-separation.test.ts index bb6d66a88e1..6533437b323 100644 --- a/tests/providers/forward-admission-separation.test.ts +++ b/tests/providers/forward-admission-separation.test.ts @@ -6,7 +6,7 @@ import { saveConfig } from "../../src/config"; import { resolveFirstUsableOpenAiSidecar } from "../../src/providers/openai-sidecar"; import { startServer } from "../../src/server"; import type { OcxConfig } from "../../src/types"; -import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../helpers/remove-tree"; const originalFetch = globalThis.fetch; const previousHome = process.env.OPENCODEX_HOME; @@ -19,6 +19,7 @@ function forwardConfig(): OcxConfig { return { port: 0, hostname: "127.0.0.1", + clientIntegrations: { codex: false }, defaultProvider: "openai", openaiProviderTierVersion: 2, providers: { @@ -49,7 +50,7 @@ beforeEach(() => { }) as typeof fetch; }); -afterEach(() => { +afterEach(async () => { globalThis.fetch = originalFetch; if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; @@ -57,9 +58,9 @@ afterEach(() => { else process.env.OPENCODEX_API_AUTH_TOKEN = previousDataToken; if (previousAdminToken === undefined) delete process.env.OPENCODEX_ADMIN_AUTH_TOKEN; else process.env.OPENCODEX_ADMIN_AUTH_TOKEN = previousAdminToken; - if (testHome) removeTreeWithRetry(testHome); + if (testHome) await removeTreeAfterBackgroundWork(testHome); testHome = ""; -}); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); describe("management credentials never leave data-plane forwarding paths", () => { test("the shared OpenAI sidecar refuses a management bearer", async () => { diff --git a/tests/providers/kiro/kiro-pool-load-settings.test.ts b/tests/providers/kiro/kiro-pool-load-settings.test.ts index e661e93c87d..a67a0250d77 100644 --- a/tests/providers/kiro/kiro-pool-load-settings.test.ts +++ b/tests/providers/kiro/kiro-pool-load-settings.test.ts @@ -1,13 +1,14 @@ -import { afterEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { createHash, randomUUID } from "node:crypto"; import { mkdtempSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { acquireAccountLease } from "../../../src/oauth/kiro-account-load"; import { clearGenericFailoverHealth, preferredInitialAccount, rotateGenericOAuthAccountOnRefusal } from "../../../src/oauth/generic-account-failover"; -import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; +import { getAccountSet, replaceProviderAccountSet } from "../../../src/oauth/store"; import { genericPoolSettingsDto, parseGenericPoolStrategy, parseKiroAccountCap, unifiedPoolSettingsDto } from "../../../src/oauth/pool-settings-capability"; import type { OcxConfig } from "../../../src/types"; -import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../../helpers/remove-tree"; import { saveConfig, loadConfig } from "../../../src/config"; import { startServer } from "../../../src/server"; import { managementFetch } from "../../helpers/management-auth"; @@ -16,29 +17,35 @@ import { installIsolatedCodexHome } from "../../helpers/isolated-codex-home"; const oldHome = process.env.OPENCODEX_HOME; let home = ""; const held: Array<{ release(): void }> = []; -afterEach(() => { +afterEach(async () => { held.splice(0).forEach(lease => lease.release()); clearGenericFailoverHealth(); if (oldHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = oldHome; - if (home) removeTreeWithRetry(home); + if (home) await removeTreeAfterBackgroundWork(home); home = ""; -}); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); async function seed(provider = "kiro") { home = mkdtempSync(join(tmpdir(), "ocx-load-settings-")); process.env.OPENCODEX_HOME = home; - for (let i = 0; i < 3; i++) await saveCredential(provider, { - access: `load-${i}`, refresh: `refresh-${i}`, expires: Date.now() + 3_600_000, - accountId: `account-${i}`, - }, { addAccount: true }); - const ids = getAccountSet(provider)!.accounts.map(row => row.id); - await setActiveAccount(provider, ids[0]!); + const now = Date.now(); + const accounts = Array.from({ length: 3 }, (_, i) => ({ + id: createHash("sha256").update(`account-${i}`).digest("hex").slice(0, 32), + credential: { access: `load-${i}`, refresh: `refresh-${i}`, expires: now + 3_600_000, accountId: `account-${i}` }, + addedAt: now, loginId: randomUUID(), + })); + // Pool behavior is under test; keep one real fenced/ACL-checked store write for fixture setup. + await replaceProviderAccountSet(provider, { activeAccountId: accounts[0]!.id, accounts }); + const stored = getAccountSet(provider)!; + const ids = stored.accounts.map(row => row.id); + expect(ids).toEqual(accounts.map(row => row.id)); + expect(stored.activeAccountId).toBe(ids[0]!); return ids; } function config(strategy: "least-loaded" | "quota" = "least-loaded", enabled = true): OcxConfig { - return { pool: { kernel: true }, providers: { kiro: { adapter: "kiro", authMode: "oauth", + return { pool: { kernel: true }, clientIntegrations: { codex: false }, providers: { kiro: { adapter: "kiro", authMode: "oauth", baseUrl: "https://runtime.us-east-1.kiro.dev", models: ["claude-sonnet-4.5"], oauthAccountFailover: { strategy, enabled, maxConcurrentPerAccount: 1 } } } } as OcxConfig; } @@ -53,15 +60,24 @@ test("Kiro reads least-loaded and maxConcurrentPerAccount in both DTOs", () => { expect(unifiedPoolSettingsDto(cfg, "anthropic", "anthropic").maxConcurrentPerAccount).toBeNull(); }); -test("Kiro persists and reads least-loaded and maxConcurrentPerAccount", async () => { - await seed(); - const isolated = installIsolatedCodexHome("ocx-load-settings-codex-"); - const saved = config("quota"); - saved.providers.xai = { adapter: "openai-chat", authMode: "oauth", baseUrl: "https://api.x.ai/v1", - models: ["test"] }; - saveConfig({ ...saved, defaultProvider: "kiro", hostname: "127.0.0.1", port: 0 } as OcxConfig); - const server = startServer(0); - try { +describe("Kiro pool settings management", () => { + let server: ReturnType | undefined; + let isolated: ReturnType | undefined; + beforeEach(async () => { + await seed(); + isolated = installIsolatedCodexHome("ocx-load-settings-codex-"); + const saved = config("quota"); + saved.providers.xai = { adapter: "openai-chat", authMode: "oauth", baseUrl: "https://api.x.ai/v1", + models: ["test"] }; + saveConfig({ ...saved, defaultProvider: "kiro", hostname: "127.0.0.1", port: 0 } as OcxConfig); + server = startServer(0); + }, 10_000); // Real Windows ACL/startup work has its own bound; the API assertions keep 5 seconds. + afterEach(async () => { + try { await server?.stop(true); } + finally { server = undefined; isolated?.restore(); isolated = undefined; } + }); + test("Kiro persists and reads least-loaded and maxConcurrentPerAccount", async () => { + if (!server) throw new Error("pool-settings fixture did not start"); const write = await managementFetch(new URL("/api/pool/settings", server.url), { method: "PUT", headers: { "content-type": "application/json" }, body: JSON.stringify({ provider: "kiro", strategy: "least-loaded", maxConcurrentPerAccount: 2 }) }); @@ -85,7 +101,7 @@ test("Kiro persists and reads least-loaded and maxConcurrentPerAccount", async ( expect(invalid.status).toBe(400); } expect(statSync(configFile).mtimeMs).toBe(before); - } finally { await server.stop(true); isolated.restore(); } + }); }); test("non-Kiro strategy and cap values are rejected by the provider parser", () => { diff --git a/tests/providers/kiro/kiro-quota-metrics.test.ts b/tests/providers/kiro/kiro-quota-metrics.test.ts index 932154526aa..0b67e0e9bce 100644 --- a/tests/providers/kiro/kiro-quota-metrics.test.ts +++ b/tests/providers/kiro/kiro-quota-metrics.test.ts @@ -1,13 +1,15 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; +import { createHash, randomUUID } from "node:crypto"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { getAccountSet, saveCredential } from "../../../src/oauth/store"; +import { getAccountSet, replaceProviderAccountSet, saveCredential } from "../../../src/oauth/store"; +import { oauthAccountLogLabel } from "../../../src/codex/account-label"; import { clearAccountQuotaCache, setCachedProviderAccountQuotaForTests } from "../../../src/providers/quota"; import { cachedKiroQuotaMetricRows } from "../../../src/providers/kiro-quota-metrics"; import { sanitizeKiroQuota } from "../../../src/providers/kiro-account-state-disk"; import { createRequestMetricsOwner } from "../../../src/server/request-metrics"; -import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS, removeTreeAfterBackgroundWork } from "../../helpers/remove-tree"; const previousHome = process.env.OPENCODEX_HOME; const realFetch = globalThis.fetch; @@ -17,13 +19,13 @@ beforeEach(() => { process.env.OPENCODEX_HOME = home; clearAccountQuotaCache(); }); -afterEach(() => { +afterEach(async () => { globalThis.fetch = realFetch; clearAccountQuotaCache(); if (previousHome === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previousHome; - removeTreeWithRetry(home); -}); + await removeTreeAfterBackgroundWork(home); +}, BACKGROUND_WORK_TEARDOWN_TIMEOUT_MS); async function account(id: string) { await saveCredential("kiro", { access: `access-${id}`, refresh: `refresh-${id}`, @@ -73,13 +75,22 @@ test("future, stale, reset-passed, and identity-mismatched rows are omitted", as test("stale leading accounts do not hide 32 later valid distinct gauges", async () => { const now = Date.now(); - for (let i = 0; i < 35; i++) { - const a = await account(String(i).padStart(3, "0")); - seed(a.id, i < 3 ? now - 31 * 60_000 : now); - } + const accounts = Array.from({ length: 35 }, (_, i) => { + const identity = String(i).padStart(3, "0"); + return { id: createHash("sha256").update(identity).digest("hex").slice(0, 32), + credential: { access: `access-${identity}`, refresh: `refresh-${identity}`, expires: now + 3_600_000, accountId: identity }, + addedAt: now, loginId: randomUUID() }; + }).sort((a, b) => a.id.localeCompare(b.id)); + // Exercise the real store once; this regression covers projection order, not repeated logins. + await replaceProviderAccountSet("kiro", { activeAccountId: accounts[34]!.id, accounts }); + const stored = getAccountSet("kiro")!.accounts; + expect(stored.map(row => row.id)).toEqual(accounts.map(row => row.id)); + expect(new Set(stored.map(row => row.credential.accountId)).size).toBe(35); + for (let i = 0; i < stored.length; i++) seed(stored[i]!.id, i < 3 ? now - 31 * 60_000 : now); const rows = cachedKiroQuotaMetricRows(now); expect(rows).toHaveLength(32); expect(new Set(rows.map(row => row.account)).size).toBe(32); + expect(rows.map(row => row.account)).toEqual(stored.slice(3).map(row => oauthAccountLogLabel(row.id, "kiro"))); }); test("duplicate or malformed labels never produce a second Prometheus series", () => { diff --git a/tests/windows/windows-tray.test.ts b/tests/windows/windows-tray.test.ts index cce64c7fafa..b1253b24dbe 100644 --- a/tests/windows/windows-tray.test.ts +++ b/tests/windows/windows-tray.test.ts @@ -652,6 +652,8 @@ describe("Windows tray packaging and command safety", () => { for (const scenario of ["Offline", "Online", "BadgeOverflowStdout", "BadgeOverflowStderr", "BadgeStaleFailure"]) { const directory = mkdtempSync(join(tmpdir(), "ocx-tray-fixture-")); const pidFile = join(directory, "pids.txt"); + // The delayed scenario holds open the empty-file window before PID publication. + if (scenario === "Online") writeFileSync(pidFile, ""); const child = Bun.spawn([process.execPath, helperPath("windows-tray-probe-child.ts"), "__startup-health"], { env: { ...process.env, OCX_PROBE_TEST_PID_FILE: pidFile, OCX_PROBE_TEST_SCENARIO: scenario }, stdout: "pipe", stderr: "pipe", @@ -670,9 +672,14 @@ describe("Windows tray packaging and command safety", () => { const stderr = drain(child.stderr); try { const deadline = Date.now() + INTERNAL_DEADLINE_MS; - while (!existsSync(pidFile) && child.exitCode === null && Date.now() < deadline) await Bun.sleep(10); + let publishedPid = ""; + do { + publishedPid = existsSync(pidFile) ? readFileSync(pidFile, "utf8").trim() : ""; + if (publishedPid === String(child.pid) || child.exitCode !== null || Date.now() >= deadline) break; + await Bun.sleep(10); + } while (true); expect(existsSync(pidFile), scenario).toBe(true); - expect(readFileSync(pidFile, "utf8").trim(), scenario).toBe(String(child.pid)); + expect(publishedPid, scenario).toBe(String(child.pid)); if (scenario.startsWith("BadgeOverflow")) { expect(await Promise.race([overflow.then(() => true), Bun.sleep(INTERNAL_DEADLINE_MS).then(() => false)]), scenario).toBe(true); } From 06764fddcde6dcfb93276e21cd97b4e51084cbe6 Mon Sep 17 00:00:00 2001 From: ZeroDi Date: Sat, 7 Nov 2026 13:03:57 +0900 Subject: [PATCH 06/26] perf(kiro): count streamed tool argument bytes incrementally (#8140) * perf(kiro): count streamed tool argument bytes incrementally * test(kiro): bound total byte-measurement input size --- scripts/test-layout/layout.json | 2 +- src/adapters/kiro/stream.ts | 9 ++- structure/transports/byte-accounting.md | 6 ++ tests/fixtures/test-layout-expected.json | 2 +- .../kiro/kiro-tool-byte-accounting.test.ts | 80 +++++++++++++++++++ 5 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 tests/providers/kiro/kiro-tool-byte-accounting.test.ts diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index c493d1db6f5..0f84a1cd193 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1100,7 +1100,7 @@ "kiro-metering-events.test.ts": "providers/kiro", "kiro-metering-usage.test.ts": "providers/kiro", "kiro-single-final.test.ts": "providers/kiro", "kiro-structured-output.test.ts": "providers/kiro", "kiro-structured-output-strict.test.ts": "providers/kiro", - "kiro-stream.test.ts": "providers/kiro", + "kiro-stream.test.ts": "providers/kiro", "kiro-tool-byte-accounting.test.ts": "providers/kiro", "kiro-transport-parity.test.ts": "providers/kiro", "kiro-usage-quota.test.ts": "providers/kiro", "kiro-usage-restart.test.ts": "providers/kiro", diff --git a/src/adapters/kiro/stream.ts b/src/adapters/kiro/stream.ts index 90c8124bb78..e49a782a1f2 100644 --- a/src/adapters/kiro/stream.ts +++ b/src/adapters/kiro/stream.ts @@ -385,7 +385,7 @@ async function* parseKiroAttemptEvents( }; } - let open: { id: string; name: string; chunks: string[]; completion: boolean } | null = null; + let open: { id: string; name: string; chunks: string[]; inputBytes: number; completion: boolean } | null = null; let openCallId: string | undefined; const closeOpenCall = () => { if (!openCallId) return; @@ -571,8 +571,8 @@ async function* parseKiroAttemptEvents( const beginTool = ( id: string, name: string, - ): { tool?: { id: string; name: string; chunks: string[]; completion: boolean }; terminal?: AdapterEvent } => { - const next = { id, name, chunks: [], completion: false }; + ): { tool?: { id: string; name: string; chunks: string[]; inputBytes: number; completion: boolean }; terminal?: AdapterEvent } => { + const next = { id, name, chunks: [], inputBytes: 0, completion: false }; const terminal = classifyTool(next); return terminal ? { terminal } : { tool: next }; }; @@ -831,12 +831,13 @@ async function* parseKiroAttemptEvents( } } if (open && ev.input !== undefined) { - const previousCallBytes = open.chunks.reduce((total, chunk) => total + Buffer.byteLength(chunk), 0); + const previousCallBytes = open.inputBytes; const nextCallBytes = previousCallBytes + Buffer.byteLength(ev.input); const callReservation = budget.reserveTransient(nextCallBytes, { kind: "tool_args", callId: open.id }); open.chunks.push(ev.input); callReservation.commitRetained(); budget.releaseRetained(previousCallBytes, { kind: "tool_args", callId: open.id }); + open.inputBytes = nextCallBytes; const nextOutputCharsBytes = appendedUtf8Bytes(outputChars, outputCharsBytes, ev.input); const toolOutputReservation = budget.reserveTransient(nextOutputCharsBytes, { kind: "retained_collectors" }); outputChars += ev.input; diff --git a/structure/transports/byte-accounting.md b/structure/transports/byte-accounting.md index 0ae0ef54ce9..4eb5fef7a5e 100644 --- a/structure/transports/byte-accounting.md +++ b/structure/transports/byte-accounting.md @@ -143,6 +143,12 @@ envelopes, and real HTTP streams; `tests/server/inbound-body-budget.test.ts` cov ## Stream-buffer accounting +Kiro tool-input collection in `src/adapters/kiro/stream.ts` keeps a per-call UTF-8 byte +total, measuring only the arriving fragment instead of remeasuring retained chunks. +Each new call starts at zero; transient replacement admission, per-call limits and +close/cancellation release remain unchanged. `tests/providers/kiro/kiro-tool-byte-accounting.test.ts` +bounds measurement work and covers Unicode byte limits and consecutive-call reset. + Command Code NDJSON in `src/adapters/command-code.ts` holds one `live_transient` charge for its decoded buffer. Growth reserves only the positive byte delta; draining releases only the removed bytes. Recounting an unchanged residual does not reserve it twice. Failed growth keeps diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index eb1492094d5..270d82a7c2f 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -965,7 +965,7 @@ "kiro-review-regressions.test.ts": "providers/kiro", "kiro-metering-events.test.ts": "providers/kiro", "kiro-metering-usage.test.ts": "providers/kiro", "kiro-completion-with-real-tools.test.ts": "providers/kiro", "kiro-single-final.test.ts": "providers/kiro", "kiro-structured-output.test.ts": "providers/kiro", "kiro-structured-output-strict.test.ts": "providers/kiro", - "kiro-stream.test.ts": "providers/kiro", "kiro-transport-parity.test.ts": "providers/kiro", + "kiro-stream.test.ts": "providers/kiro", "kiro-tool-byte-accounting.test.ts": "providers/kiro", "kiro-transport-parity.test.ts": "providers/kiro", "kiro-usage-quota.test.ts": "providers/kiro", "kiro-usage-restart.test.ts": "providers/kiro", "kiro-windows-cli-db-path.test.ts": "providers/kiro", "kiro-windows-cli-executable-path.test.ts": "providers/kiro", "kiro-wire-estimate.test.ts": "providers/kiro", "lab-activation.test.ts": "lab", diff --git a/tests/providers/kiro/kiro-tool-byte-accounting.test.ts b/tests/providers/kiro/kiro-tool-byte-accounting.test.ts new file mode 100644 index 00000000000..08ba068ba40 --- /dev/null +++ b/tests/providers/kiro/kiro-tool-byte-accounting.test.ts @@ -0,0 +1,80 @@ +import { expect, spyOn, test } from "bun:test"; +import { parseKiroStream } from "../../../src/adapters/kiro/stream"; +import { encodeMessage } from "../../../src/lib/eventstream-decoder"; +import { createTranslatorBudget } from "../../../src/lib/translator-budget"; +import type { AdapterEvent } from "../../../src/types"; + +/** Exercise the real Smithy decoder and Kiro parser without a provider request. */ +function responseFor(calls: string[][]): Response { + const encoder = new TextEncoder(); + const frames = calls.flatMap((chunks, index) => [ + { name: "write_file", toolUseId: `call-${index}` }, + ...chunks.map(input => ({ input })), + { stop: true }, + ]).map(event => encodeMessage( + { ":message-type": "event", ":event-type": "toolUseEvent" }, + encoder.encode(JSON.stringify(event)), + )); + let index = 0; + return new Response(new ReadableStream({ + pull(controller) { + if (index < frames.length) controller.enqueue(frames[index++]); + else controller.close(); + }, + })); +} + +test("fragmented Kiro tool arguments require linear UTF-8 measurement work", async () => { + const fragment = "x".repeat(256); + const count = 256; + const chunks = ['{"value":"', ...Array(count).fill(fragment), '"}']; + const response = responseFor([chunks]); + const byteLength = Buffer.byteLength; + let measurements = 0; + let measuredChars = 0; + const measuring = spyOn(Buffer, "byteLength").mockImplementation((...args: Parameters) => { + if (args[0] === fragment) measurements++; + if (typeof args[0] === "string") measuredChars += args[0].length; + return Reflect.apply(byteLength, Buffer, args); + }); + const budget = createTranslatorBudget(); + const events: AdapterEvent[] = []; + try { + for await (const event of parseKiroStream(response, budget)) events.push(event); + expect(events.filter(event => event.type === "tool_call_delta").map(event => event.arguments).join("")).toBe(chunks.join("")); + expect(events.at(-1)?.type).toBe("done"); + expect(budget.snapshot().currentBytes).toBe(0); + expect(budget.snapshot().activeCalls).toBe(0); + // One measurement for call retention and one for output accounting; leave bounded headroom. + expect(measurements).toBeLessThanOrEqual(3 * count); + // Include growing-prefix measurements, with headroom for serialized adapter events. + expect(measuredChars).toBeLessThanOrEqual(16 * chunks.join("").length); + } finally { + measuring.mockRestore(); + budget.dispose(); + } +}); + +test("Kiro argument byte totals reset per call and retain exact UTF-8 limits", async () => { + const chunks = ['{"value":"', "한글😀", '"}']; + const bytes = chunks.reduce((total, chunk) => total + Buffer.byteLength(chunk), 0); + for (const limit of [bytes - 1, bytes, bytes + 1]) { + const budget = createTranslatorBudget({ maxCallArgumentBytes: limit }); + const events: AdapterEvent[] = []; + try { + for await (const event of parseKiroStream(responseFor([chunks, chunks]), budget)) events.push(event); + if (limit < bytes) { + expect(events.some(event => event.type === "error")).toBe(true); + expect(events.some(event => event.type === "tool_call_start")).toBe(false); + } else { + expect(events.filter(event => event.type === "tool_call_start").map(event => event.id)).toEqual(["call-0", "call-1"]); + expect(events.filter(event => event.type === "tool_call_delta").map(event => event.arguments).join("")).toBe(chunks.join("").repeat(2)); + expect(events.at(-1)?.type).toBe("done"); + } + expect(budget.snapshot().currentBytes).toBe(0); + expect(budget.snapshot().activeCalls).toBe(0); + } finally { + budget.dispose(); + } + } +}); From af92fb4e9e9d8023cd65153ecc4c4793593eec61 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sat, 7 Nov 2026 13:04:04 +0900 Subject: [PATCH 07/26] fix(ollama): bound tool-call validation work (#8147) --- .../content/docs/fr/guides/ollama-clients.md | 5 +++ .../src/content/docs/guides/ollama-clients.md | 5 +++ .../content/docs/ja/guides/ollama-clients.md | 4 ++ .../content/docs/ko/guides/ollama-clients.md | 4 ++ .../content/docs/ru/guides/ollama-clients.md | 5 +++ .../content/docs/tr/guides/ollama-clients.md | 5 +++ .../docs/zh-cn/guides/ollama-clients.md | 4 ++ .../docs/zh-tw/guides/ollama-clients.md | 4 ++ src/protocols/codecs/ollama.ts | 34 +++++++++------ structure/data-planes/ollama-inbound.md | 4 +- .../responses/responses-ollama-codec.test.ts | 43 ++++++++++++++++++- tests/server/server-ollama-inbound.test.ts | 17 ++++++++ 12 files changed, 120 insertions(+), 14 deletions(-) diff --git a/docs-site/src/content/docs/fr/guides/ollama-clients.md b/docs-site/src/content/docs/fr/guides/ollama-clients.md index a3e78b7a28f..58c132f1f2e 100644 --- a/docs-site/src/content/docs/fr/guides/ollama-clients.md +++ b/docs-site/src/content/docs/fr/guides/ollama-clients.md @@ -44,6 +44,11 @@ Tous les autres chemins `/api/*` restent dans l'API de gestion d'opencodex. - `options` : `temperature`, `top_p`, `num_predict` et `stop`. - `format` : uniquement `"json"`. +Chaque requête `/api/chat` accepte au maximum **1 024 appels d’outils au total** dans l’ensemble des messages +assistant, y compris ceux qui ont déjà un résultat. Au-delà, elle renvoie HTTP 400 avant d’atteindre +un fournisseur. Raccourcissez l’historique de la conversation et renvoyez-le, en conservant chaque +résultat d’outil avec l’appel correspondant dans le message assistant. + Tout le reste est refusé avec un HTTP 400 au lieu d'être ignoré : `think`, `keep_alive`, `suffix`, `raw`, `template`, `context`, les autres `options`, un `format` en schéma JSON et les URL d'images. Les points de terminaison Ollama de gestion des modèles et d'embeddings ne sont pas fournis. diff --git a/docs-site/src/content/docs/guides/ollama-clients.md b/docs-site/src/content/docs/guides/ollama-clients.md index 9875a96a051..7e4f1534cad 100644 --- a/docs-site/src/content/docs/guides/ollama-clients.md +++ b/docs-site/src/content/docs/guides/ollama-clients.md @@ -45,6 +45,11 @@ Every other `/api/*` path stays part of the opencodex management API. - `options`: `temperature`, `top_p`, `num_predict` and `stop`. - `format`: `"json"` only. +Each `/api/chat` request accepts at most **1,024 tool calls in total** across all assistant messages, +including calls that already have tool results. Above this limit, the request returns HTTP 400 +before reaching a provider. Shorten the conversation history and resend it, keeping each retained +tool result together with its corresponding assistant tool call. + Anything else is rejected with HTTP 400 instead of being ignored. That includes `think`, `keep_alive`, `suffix`, `raw`, `template`, `context`, other `options`, JSON-schema `format` and image URLs. Ollama's model-management and embedding endpoints are not provided. diff --git a/docs-site/src/content/docs/ja/guides/ollama-clients.md b/docs-site/src/content/docs/ja/guides/ollama-clients.md index da7f99a19a8..fb6f55f26fc 100644 --- a/docs-site/src/content/docs/ja/guides/ollama-clients.md +++ b/docs-site/src/content/docs/ja/guides/ollama-clients.md @@ -43,6 +43,10 @@ curl "$OLLAMA_HOST/api/tags" - `options`: `temperature`、`top_p`、`num_predict`、`stop`。 - `format`: `"json"` のみ。 +各 `/api/chat` リクエストでは、すべての assistant メッセージを合わせて **ツール呼び出しは最大 1,024 件**です。 +結果がすでに返された呼び出しも含みます。上限を超えると、プロバイダーに送信される前に HTTP 400 が返されます。 +会話履歴を短くして再送してください。残すツール結果と、それに対応する assistant のツール呼び出しは必ず一緒に残します。 + それ以外は無視されず、HTTP 400 で拒否されます。`think`、`keep_alive`、`suffix`、`raw`、`template`、 `context`、その他の `options`、JSON スキーマ形式の `format`、画像 URL が該当します。Ollama のモデル管理および 埋め込みエンドポイントは提供されません。 diff --git a/docs-site/src/content/docs/ko/guides/ollama-clients.md b/docs-site/src/content/docs/ko/guides/ollama-clients.md index 3ffd8c49749..26bb2eb6a2b 100644 --- a/docs-site/src/content/docs/ko/guides/ollama-clients.md +++ b/docs-site/src/content/docs/ko/guides/ollama-clients.md @@ -43,6 +43,10 @@ curl "$OLLAMA_HOST/api/tags" - `options`: `temperature`, `top_p`, `num_predict`, `stop`. - `format`: `"json"`만 지원합니다. +각 `/api/chat` 요청은 모든 assistant 메시지에 걸쳐 **도구 호출을 합계 1,024개까지** 허용합니다. +이미 결과가 있는 호출도 합계에 포함됩니다. 한도를 넘으면 프로바이더로 보내기 전에 HTTP 400을 반환합니다. +대화 이력을 줄인 뒤 다시 보내세요. 남겨 둔 도구 결과에는 해당 assistant 도구 호출도 함께 남겨야 합니다. + 그 밖의 항목은 무시되지 않고 HTTP 400으로 거부됩니다. `think`, `keep_alive`, `suffix`, `raw`, `template`, `context`, 기타 `options`, JSON 스키마 `format`, 이미지 URL이 여기에 해당합니다. Ollama의 모델 관리 및 임베딩 엔드포인트는 제공되지 않습니다. diff --git a/docs-site/src/content/docs/ru/guides/ollama-clients.md b/docs-site/src/content/docs/ru/guides/ollama-clients.md index 0bef2a6a995..7fde2f37832 100644 --- a/docs-site/src/content/docs/ru/guides/ollama-clients.md +++ b/docs-site/src/content/docs/ru/guides/ollama-clients.md @@ -43,6 +43,11 @@ curl "$OLLAMA_HOST/api/tags" - `options`: `temperature`, `top_p`, `num_predict` и `stop`. - `format`: только `"json"`. +Каждый запрос `/api/chat` допускает **не более 1 024 вызовов инструментов суммарно** во всех сообщениях +assistant, включая вызовы, для которых уже есть результаты. При превышении лимита запрос возвращает +HTTP 400 до отправки провайдеру. Сократите историю диалога и отправьте её снова, сохраняя каждый +оставленный результат инструмента вместе с соответствующим вызовом в сообщении assistant. + Всё остальное отклоняется с HTTP 400, а не игнорируется: `think`, `keep_alive`, `suffix`, `raw`, `template`, `context`, прочие `options`, `format` в виде JSON-схемы и URL изображений. Конечные точки Ollama для управления моделями и эмбеддингов не предоставляются. diff --git a/docs-site/src/content/docs/tr/guides/ollama-clients.md b/docs-site/src/content/docs/tr/guides/ollama-clients.md index 6921e7b67b7..8409b46a1fd 100644 --- a/docs-site/src/content/docs/tr/guides/ollama-clients.md +++ b/docs-site/src/content/docs/tr/guides/ollama-clients.md @@ -43,6 +43,11 @@ Diğer tüm `/api/*` yolları opencodex yönetim API'sinin parçası olarak kal - `options`: `temperature`, `top_p`, `num_predict` ve `stop`. - `format`: yalnızca `"json"`. +Her `/api/chat` isteği, tüm assistant mesajlarında **toplam en fazla 1.024 araç çağrısı** kabul eder; +sonucu zaten bulunan çağrılar da bu sayıya dahildir. Sınır aşılırsa istek sağlayıcıya ulaşmadan HTTP 400 +döndürür. Konuşma geçmişini kısaltıp yeniden gönderin; tuttuğunuz her araç sonucuyla birlikte ilgili +assistant araç çağrısını da koruyun. + Bunların dışındaki her şey yok sayılmak yerine HTTP 400 ile reddedilir: `think`, `keep_alive`, `suffix`, `raw`, `template`, `context`, diğer `options`, JSON şeması biçimindeki `format` ve görsel URL'leri. Ollama'nın model yönetimi ve embedding uç noktaları sağlanmaz. diff --git a/docs-site/src/content/docs/zh-cn/guides/ollama-clients.md b/docs-site/src/content/docs/zh-cn/guides/ollama-clients.md index 2701bc4d13d..14999cb4126 100644 --- a/docs-site/src/content/docs/zh-cn/guides/ollama-clients.md +++ b/docs-site/src/content/docs/zh-cn/guides/ollama-clients.md @@ -42,6 +42,10 @@ curl "$OLLAMA_HOST/api/tags" - `options`：`temperature`、`top_p`、`num_predict` 和 `stop`。 - `format`：仅支持 `"json"`。 +每个 `/api/chat` 请求中，所有 assistant 消息的**工具调用总数最多为 1,024 次**，已返回结果的调用也计入总数。 +超过上限时，请求会在发送给提供商之前返回 HTTP 400。请缩短对话历史后重试，并将保留的工具结果与对应的 +assistant 工具调用一起保留。 + 其他内容会以 HTTP 400 拒绝，而不是被忽略，包括 `think`、`keep_alive`、`suffix`、`raw`、 `template`、`context`、其他 `options`、JSON Schema 形式的 `format` 以及图片 URL。不提供 Ollama 的模型管理和 嵌入端点。 diff --git a/docs-site/src/content/docs/zh-tw/guides/ollama-clients.md b/docs-site/src/content/docs/zh-tw/guides/ollama-clients.md index fb4cf878e5b..af93baf37cc 100644 --- a/docs-site/src/content/docs/zh-tw/guides/ollama-clients.md +++ b/docs-site/src/content/docs/zh-tw/guides/ollama-clients.md @@ -42,6 +42,10 @@ curl "$OLLAMA_HOST/api/tags" - `options`：`temperature`、`top_p`、`num_predict` 與 `stop`。 - `format`：僅支援 `"json"`。 +每個 `/api/chat` 請求中，所有 assistant 訊息的**工具呼叫總數最多為 1,024 次**，已傳回結果的呼叫也計入總數。 +超過上限時，請求會在傳送給供應商之前回傳 HTTP 400。請縮短對話歷史後重試，並將保留的工具結果與對應的 +assistant 工具呼叫一起保留。 + 其他內容會以 HTTP 400 拒絕，而不是被忽略，包括 `think`、`keep_alive`、`suffix`、`raw`、 `template`、`context`、其他 `options`、JSON Schema 形式的 `format` 以及圖片 URL。不提供 Ollama 的模型管理與 嵌入端點。 diff --git a/src/protocols/codecs/ollama.ts b/src/protocols/codecs/ollama.ts index 85c1c7d151e..a02128f8cd8 100644 --- a/src/protocols/codecs/ollama.ts +++ b/src/protocols/codecs/ollama.ts @@ -1,6 +1,8 @@ /** Pure Ollama native request and Chat response projections. */ type Obj = Record; export type OllamaKind = "chat" | "generate"; +export const MAX_OLLAMA_TOOL_CALLS = 1_024; +type PendingCall = { id: string; name: string; group: Set; matched: boolean }; export class OllamaCodecError extends Error { readonly status = 400; readonly code = "invalid_ollama_request"; @@ -63,7 +65,9 @@ export function ollamaToChat(kind: OllamaKind, input: unknown): Obj { messages.push({ role: "user", content: content(body.prompt, body.images) }); } else { if (!Array.isArray(body.messages)) fail("Invalid messages"); - const pending: { id: string; name: string; message: number; matched: boolean }[] = []; + const pendingById = new Map(); + const pendingByName = new Map[]>(); + let callCount = 0; body.messages.forEach((raw, mi) => { const msg = object(raw, "message"); const role = string(msg.role, "role"); @@ -72,30 +76,36 @@ export function ollamaToChat(kind: OllamaKind, input: unknown): Obj { const projected: Obj = { role, content: content(msg.content, role === "user" ? msg.images : undefined) }; if (role === "assistant" && msg.tool_calls !== undefined) { if (!Array.isArray(msg.tool_calls)) fail("Invalid tool calls"); + if (callCount + msg.tool_calls.length > MAX_OLLAMA_TOOL_CALLS) fail("Too many tool calls"); + callCount += msg.tool_calls.length; + const groups = new Map>(); projected.tool_calls = msg.tool_calls.map((rawCall, ci) => { // A replayed response call keeps the id this proxy returned for it; older clients send none. const call = object(rawCall, "tool call"); keys(call, ["id", "function"], "tool call"); const fn = object(call.function, "tool call function"); keys(fn, ["name", "arguments"], "tool call function"); const name = string(fn.name, "tool name", true); const args = object(fn.arguments, "tool arguments"); const id = call.id === undefined ? `call_ollama_${mi}_${ci}` : string(call.id, "tool call id", true); - if (pending.some(p => p.id === id)) fail("Duplicate tool call id"); - pending.push({ id, name, message: mi, matched: false }); + if (pendingById.has(id)) fail("Duplicate tool call id"); + const group = groups.get(name) ?? new Set(); groups.set(name, group); + const pending: PendingCall = { id, name, group, matched: false }; group.add(pending); pendingById.set(id, pending); return { id, type: "function", function: { name, arguments: JSON.stringify(args) } }; }); + for (const [name, group] of groups) { + const named = pendingByName.get(name) ?? []; named.push(group); pendingByName.set(name, named); + } } if (role === "tool") { const id = msg.tool_call_id === undefined ? undefined : string(msg.tool_call_id, "tool call id", true); const name = msg.tool_name === undefined ? undefined : string(msg.tool_name, "tool name", true); - const candidates = pending.filter(p => !p.matched && (id ? p.id === id && (!name || p.name === name) : name && p.name === name)); - if (!id && name && candidates.length) { - const latestMessage = candidates[candidates.length - 1]!.message; - const latest = candidates.filter(p => p.message === latestMessage); - if (latest.length !== 1) fail("Ambiguous tool result"); - latest[0]!.matched = true; projected.tool_call_id = latest[0]!.id; - } else { - if (candidates.length !== 1) fail("Unmatched tool result"); - candidates[0]!.matched = true; projected.tool_call_id = candidates[0]!.id; + let candidate: PendingCall | undefined; + if (id) candidate = pendingById.get(id); + else if (name) { + const groups = pendingByName.get(name); while (groups?.length && !groups.at(-1)!.size) groups.pop(); + const group = groups?.at(-1); if (group && group.size !== 1) fail("Ambiguous tool result"); + candidate = group?.values().next().value; } + if (!candidate || candidate.matched || (name && candidate.name !== name)) fail("Unmatched tool result"); + candidate.matched = true; candidate.group.delete(candidate); projected.tool_call_id = candidate.id; } messages.push(projected); }); diff --git a/structure/data-planes/ollama-inbound.md b/structure/data-planes/ollama-inbound.md index cae06d0e81a..483d17811c2 100644 --- a/structure/data-planes/ollama-inbound.md +++ b/structure/data-planes/ollama-inbound.md @@ -29,7 +29,9 @@ key's model scope. Show returns no file, digest, template or license facts. `src/protocols/codecs/ollama.ts` validates the native body (unknown fields and options, `think`, `keep_alive`, non-JSON formats and remote images are 400s) and projects it to Chat; base64 images -become Chat `image_url` data URLs with the sniffed media type. `src/server/ollama-inbound.ts` calls +become Chat `image_url` data URLs with the sniffed media type. Chat requests accept at most 1,024 +tool calls in aggregate, and tool-call IDs and pending results are indexed so validation work stays +linear in the request size. `src/server/ollama-inbound.ts` calls `handleChatCompletions` with a synthesized request carrying only the content type, caller session and workflow identity headers (`session_id`, `session-id`, `thread-id`, `x-session-id`, `x-codex-parent-thread-id`, so the workflow root's spend ceiling still applies) and the client diff --git a/tests/responses/responses-ollama-codec.test.ts b/tests/responses/responses-ollama-codec.test.ts index f107706f405..dd066fa7986 100644 --- a/tests/responses/responses-ollama-codec.test.ts +++ b/tests/responses/responses-ollama-codec.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { ollamaToChat, ollamaBufferedResponse, createOllamaStreamState, consumeOllamaChatChunk, ollamaStreamTerminal, OllamaCodecError } from "../../src/protocols/codecs/ollama"; +import { ollamaToChat, ollamaBufferedResponse, createOllamaStreamState, consumeOllamaChatChunk, ollamaStreamTerminal, OllamaCodecError, MAX_OLLAMA_TOOL_CALLS } from "../../src/protocols/codecs/ollama"; const png = "iVBORw0KGgo="; const schema = { type: "object", properties: { q: { type: "string", enum: ["a", "b"] } }, required: ["q"], additionalProperties: false }; @@ -40,6 +40,47 @@ describe("Ollama codec", () => { expect(() => ollamaToChat("chat", { model: "m", messages: duplicate })).toThrow(OllamaCodecError); expect(() => ollamaToChat("chat", { model: "m", messages: [{ role: "assistant", content: "", tool_calls: [{ id: 7, function: { name: "f", arguments: {} } }] }] })).toThrow(OllamaCodecError); }); + test("bounds aggregate tool-call validation work", () => { + expect(MAX_OLLAMA_TOOL_CALLS).toBe(1_024); + const calls = Array.from({ length: MAX_OLLAMA_TOOL_CALLS }, (_, i) => ({ id: `id${i}`, function: { name: "f", arguments: {} } })); + expect((ollamaToChat("chat", { model: "m", messages: [{ role: "assistant", content: "", tool_calls: calls }] }).messages as any[])[0].tool_calls).toHaveLength(MAX_OLLAMA_TOOL_CALLS); + calls.push({ id: "overflow", function: { name: "f", arguments: {} } }); + expect(() => ollamaToChat("chat", { model: "m", messages: [{ role: "assistant", content: "", tool_calls: calls }] })).toThrow("Too many tool calls"); + }); + test("counts calls across assistant messages even after their results are matched", () => { + const calls = Array.from({ length: 1_024 }, (_, i) => ({ id: `id${i}`, function: { name: "f", arguments: {} } })); + const laterCalls = calls.slice(512); + const messages = [ + { role: "assistant", content: "", tool_calls: calls.slice(0, 512) }, + ...calls.slice(0, 512).map(call => ({ role: "tool", content: "done", tool_call_id: call.id })), + { role: "assistant", content: "", tool_calls: laterCalls }, + ]; + const accepted = ollamaToChat("chat", { model: "m", messages }).messages as { tool_calls?: unknown[] }[]; + expect(accepted.at(-1)!.tool_calls).toHaveLength(512); + laterCalls.push({ id: "overflow", function: { name: "f", arguments: {} } }); + expect(() => ollamaToChat("chat", { model: "m", messages })).toThrow("Too many tool calls"); + }); + test("matches the latest unambiguous name group before falling back to earlier calls", () => { + const call = (id: string) => ({ id, function: { name: "f", arguments: {} } }); + const history = [ + { role: "assistant", content: "", tool_calls: [call("older")] }, + { role: "assistant", content: "", tool_calls: [call("newer-a"), call("newer-b")] }, + ]; + const byName = { role: "tool", content: "done", tool_name: "f" }; + expect(() => ollamaToChat("chat", { model: "m", messages: [...history, byName] })).toThrow("Ambiguous tool result"); + const messages = [...history, { role: "tool", content: "done", tool_call_id: "newer-b" }, byName, byName]; + const matched = ollamaToChat("chat", { model: "m", messages }).messages as { tool_call_id?: string }[]; + expect(matched.slice(2).map(message => message.tool_call_id)).toEqual(["newer-b", "newer-a", "older"]); + }); + test("keeps duplicate IDs reserved after a result and refuses mismatched explicit IDs", () => { + const assistant = { role: "assistant", content: "", tool_calls: [{ id: "stable", function: { name: "f", arguments: {} } }] }; + const result = { role: "tool", content: "done", tool_call_id: "stable" }; + expect(() => ollamaToChat("chat", { model: "m", messages: [assistant, result, assistant] })).toThrow("Duplicate tool call id"); + for (const unmatched of [ + { ...result, tool_name: "other" }, + { ...result, tool_call_id: "missing", tool_name: "f" }, + ]) expect(() => ollamaToChat("chat", { model: "m", messages: [assistant, unmatched] })).toThrow("Unmatched tool result"); + }); test("rejects unsupported and malformed shapes with bounded errors", () => { for (const body of [ { model: "m", prompt: "x", suffix: "y" }, { model: "m", prompt: "x", think: false }, diff --git a/tests/server/server-ollama-inbound.test.ts b/tests/server/server-ollama-inbound.test.ts index 057583ddf50..ce9835ba701 100644 --- a/tests/server/server-ollama-inbound.test.ts +++ b/tests/server/server-ollama-inbound.test.ts @@ -234,6 +234,23 @@ describe("Ollama-compatible inbound routes", () => { }); } + test("chat refuses aggregate tool-call overflow before any upstream send", async () => { + const calls = Array.from({ length: 1_025 }, (_, i) => ({ id: `call_${i}`, function: { name: "lookup", arguments: {} } })); + await withServer(async base => { + for (const stream of [false, true]) { + const res = await post(base, "/api/chat", { model: "fx/m1", stream, messages: [ + { role: "assistant", content: "", tool_calls: calls.slice(0, 512) }, + { role: "assistant", content: "", tool_calls: calls.slice(512) }, + ] }); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ error: "Too many tool calls" }); + expect(rowFor(res)!.status).toBe(400); + } + expect(seen).toHaveLength(0); + expect(newRows()).toHaveLength(2); + }); + }); + test("chat tools round trip with stable call ids", async () => { mode = "tools"; await withServer(async base => { From f1ce135e80523ae1cf36be849ada4fd572189409 Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sat, 7 Nov 2026 13:10:17 +0900 Subject: [PATCH 08/26] fix(responses): preserve mandatory tool choices through routing and recovery (#8146) Preserve mandatory tool choices through routing and account recovery. Co-authored-by: Epinephrine --- .../content/docs/reference/proxy-formats.md | 12 + scripts/test-layout/layout-responses.json | 1 + src/server/request-log-attempt-recovery.ts | 17 + src/server/request-log.ts | 23 +- .../responses/account-auto-pause-recovery.ts | 30 +- src/server/responses/adapter-continuation.ts | 17 +- src/server/responses/adapter-dispatch.ts | 53 ++- .../responses/combo-required-tool-contract.ts | 25 +- src/server/responses/core-combo-failure.ts | 2 + src/server/responses/core-combo.ts | 13 +- src/server/responses/core-lifetime.ts | 3 + .../responses/deferred-send-reporter.ts | 19 +- src/server/responses/fetch-helpers.ts | 8 +- src/server/responses/passthrough-delivery.ts | 5 +- src/server/responses/passthrough-dispatch.ts | 70 +-- src/server/responses/policy-fallback.ts | 37 +- src/server/responses/policy-request-scope.ts | 26 ++ src/server/responses/request-prepare.ts | 13 +- src/server/responses/request-send-budget.ts | 4 +- src/server/responses/request-spend-target.ts | 19 + src/server/responses/request-transport.ts | 155 ++++--- structure/dashboard-and-usage.md | 2 +- structure/transports/policy-fallback.md | 38 ++ .../transports/responses-combo-boundaries.md | 15 + .../translator-budget-lifetime.test.ts | 47 ++ tests/fixtures/test-layout-expected.json | 3 +- .../anthropic-billing-send-budget-driver.ts | 3 + tests/helpers/responses-core-source.ts | 1 + .../github-copilot-auto-runtime.test.ts | 409 +++++++++++++++++- .../responses-account-auto-pause.test.ts | 92 +++- .../responses-direct-chat-tool-choice.test.ts | 203 +++++++++ .../responses-fetch-helpers-boundary.test.ts | 21 + ...onses-passthrough-transient-policy.test.ts | 10 +- tests/routing/routing-policy-fallback.test.ts | 293 ++++++++++++- ...erver-combo-required-tool-contract.test.ts | 45 ++ 35 files changed, 1551 insertions(+), 183 deletions(-) create mode 100644 src/server/request-log-attempt-recovery.ts create mode 100644 src/server/responses/request-spend-target.ts create mode 100644 tests/responses/responses-direct-chat-tool-choice.test.ts diff --git a/docs-site/src/content/docs/reference/proxy-formats.md b/docs-site/src/content/docs/reference/proxy-formats.md index b17aaec2094..4786c484b00 100644 --- a/docs-site/src/content/docs/reference/proxy-formats.md +++ b/docs-site/src/content/docs/reference/proxy-formats.md @@ -50,6 +50,18 @@ snapshot omits it. Auto/none, incomplete results and unknown contracts retain th ## Required tools on translated Chat routes +Before sending, the proxy rejects a mandatory tool choice that the selected Chat destination cannot +represent, such as `computer_use` or a required allowed-tool list containing MCP selections. A direct +route returns HTTP 400 without an upstream request. A routing profile can continue to another +originally eligible destination under its configured fallback policy, preserving the original choice. +An explicit nonempty catalog with `required` is also refused when no tool survives in the full parsed Chat catalog, including tools supplied through input items. A surviving client tool keeps a mixed catalog eligible; hosted web search retains its later sidecar handling. +If no eligible target remains and all attempted candidates were locally incompatible, the profile returns the same HTTP 400 diagnostic; +the Responses API returns HTTP 503 for an exhausted combo, naming the mandatory-tool constraint. Other failures +retain their existing error handling. Copilot Auto uses the final negotiated model and wire for this check. +A local incompatibility before inference does not count as a model send, including after a session refresh. +Negotiating a replacement account or model does not rewrite the attribution of an earlier physical send. +If a send-budget limit stops traversal first, the proxy retains the preceding failure instead of blaming tool compatibility. + A translated OpenAI Chat response must contain a complete selected client tool call when `tool_choice` is required, names a tool, or uses a required allowed-tool selection. If the upstream finishes normally without one, the turn fails with `required_tool_not_called` and preserves usage. diff --git a/scripts/test-layout/layout-responses.json b/scripts/test-layout/layout-responses.json index 3887dc23ccf..1e6dd54f880 100644 --- a/scripts/test-layout/layout-responses.json +++ b/scripts/test-layout/layout-responses.json @@ -1,4 +1,5 @@ { + "responses-direct-chat-tool-choice.test.ts": "responses", "responses-native-output-identity.test.ts": "responses", "graceful-max-output-tokens.test.ts": "responses", "graceful-max-output-tokens-exclusions.test.ts": "responses", diff --git a/src/server/request-log-attempt-recovery.ts b/src/server/request-log-attempt-recovery.ts new file mode 100644 index 00000000000..400f0e6426d --- /dev/null +++ b/src/server/request-log-attempt-recovery.ts @@ -0,0 +1,17 @@ +import type { AttemptRecoveryWithheld, PersistedUsageAttempt } from "../usage/log"; + +/** + * Record that a recovery this attempt was eligible for did not happen. + * + * Deliberately NOT `noteAttemptSend`: nothing was sent, so `sendCount` must not move. The two + * together are what make a one-send log readable — no kind and no withheld reason means nothing + * was eligible, a withheld reason means something was and the budget refused it (#5044). + */ +export function noteAttemptRecoveryWithheld( + attempt: PersistedUsageAttempt | undefined, + reason: AttemptRecoveryWithheld, +): void { + if (!attempt) return; + if (!attempt.recoveryWithheld) attempt.recoveryWithheld = []; + if (!attempt.recoveryWithheld.includes(reason)) attempt.recoveryWithheld.push(reason); +} diff --git a/src/server/request-log.ts b/src/server/request-log.ts index f9ae1e62d75..03668088068 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -5,6 +5,7 @@ import { stampApiKeyAccountLabel, usesApiKeyAccount } from "../providers/label"; import { KEY_ACCOUNT_LOG_LABEL_RE } from "../codex/account-label"; import { attemptAccountChanged, sealRequestAttemptIdentity } from "./request-log-account-rotation"; export { sealRequestAttemptIdentity }; +export { noteAttemptRecoveryWithheld } from "./request-log-attempt-recovery"; import { captureUpstreamTerminalDiagnostics, upstreamDiagnosticLogFields, type UpstreamErrorDiagnostics, type UpstreamErrorPayload } from "./request-log-terminal-status"; export { noteUpstreamRequestId } from "./request-log-terminal-status"; import { readBoundedResponseBody } from "../lib/bounded-body"; @@ -58,7 +59,6 @@ import { usageStatusForFinalLog, usageTotalTokens, type AttemptRecoveryKind, - type AttemptRecoveryWithheld, type CacheTelemetryProvenance, type PersistedRequestSpend, type PersistedUsageAttempt, @@ -1789,13 +1789,15 @@ export function noteProviderAttemptSend( provider: OcxProviderConfig, inputTokenEstimate: number | undefined, recovery?: AttemptRecoveryKind, + resolved?: { model: string; adapter: string }, ): void { const attempt = logCtx.activeAttempt; const previous = attempt?.accountLogLabel; stampApiKeyAccountLabel(logCtx, providerName, provider); const next = logCtx.accountLogLabel; if (attempt && usesApiKeyAccount(provider)) keyUsageOwners.add(attempt); - if (attempt && attempt.sendCount > 0 && attemptAccountChanged(previous, next, attempt.provider, logCtx.provider)) { + if (attempt && attempt.sendCount > 0 && (attemptAccountChanged(previous, next, attempt.provider, logCtx.provider) + || resolved && (attempt.model !== resolved.model || attempt.adapter !== resolved.adapter))) { // An input estimate is not evidence that a failed send used that many tokens. delete attempt.inputTokenEstimate; // An account-scoped recovery kind names the status the sealed account answered. @@ -1827,6 +1829,7 @@ export function noteProviderAttemptSend( logCtx.activeAttemptStartedAt = Date.now(); } if (attempt) { + if (resolved) { attempt.model = resolved.model; attempt.adapter = resolved.adapter; } sealRequestAttemptIdentity(attempt, logCtx.provider, attempt.adapter, next); recordAttemptCredentialSource(attempt, providerName, provider, attempt.adapter); } @@ -1885,22 +1888,6 @@ export function noteAttemptSend( } } -/** - * Record that a recovery this attempt was eligible for did not happen. - * - * Deliberately NOT `noteAttemptSend`: nothing was sent, so `sendCount` must not move. The two - * together are what make a one-send log readable — no kind and no withheld reason means nothing - * was eligible, a withheld reason means something was and the budget refused it (#5044). - */ -export function noteAttemptRecoveryWithheld( - attempt: PersistedUsageAttempt | undefined, - reason: AttemptRecoveryWithheld, -): void { - if (!attempt) return; - if (!attempt.recoveryWithheld) attempt.recoveryWithheld = []; - if (!attempt.recoveryWithheld.includes(reason)) attempt.recoveryWithheld.push(reason); -} - export function finishRequestAttempt( attempt: PersistedUsageAttempt, status: number, diff --git a/src/server/responses/account-auto-pause-recovery.ts b/src/server/responses/account-auto-pause-recovery.ts index 0bd39b64f8d..5c1f6538259 100644 --- a/src/server/responses/account-auto-pause-recovery.ts +++ b/src/server/responses/account-auto-pause-recovery.ts @@ -1,5 +1,24 @@ import type { ResponsesSendBudget } from "./request-send-budget"; +/** Dispose the failed exchange before projecting a refresh refusal or rethrowing its original error. */ +export function createAutoPauseRefreshErrorMapper( + response: Response, + upstream: { abort(): void }, + cleanup: () => void, + refusal: (error: unknown) => Response | undefined, + cancelled: () => boolean, + cancelledResponse: () => Response, +): (error: unknown) => { failed: Response } | undefined { + return error => { + const clientCancelled = cancelled(); + try { void response.body?.cancel().catch(() => {}); } catch { /* closed */ } + cleanup(); + upstream.abort(); + const failed = clientCancelled ? cancelledResponse() : refusal(error); + return failed ? { failed } : undefined; + }; +} + const moves = new WeakMap(); /** One same-provider credential move, funded by the existing recovery owner. */ export async function recoverAutoPausedAccount(options: { @@ -7,6 +26,8 @@ export async function recoverAutoPausedAccount(options: { remaining?: number; reserve: ResponsesSendBudget["reserveCredentialHop"]; refresh: () => Promise; + /** Map a known refresh refusal; unknown errors still propagate after unused-hop cleanup. */ + mapRefreshError?: (error: unknown) => T | undefined; refetch: () => Promise; setPermit: (permit: ResponsesSendBudget["pendingHopPermit"]) => void; }): Promise { @@ -15,7 +36,14 @@ export async function recoverAutoPausedAccount(options: { const hop = options.reserve("auth-recovery", "account-auto-pause", true); if (!hop.allowed) return; try { - if (!await options.refresh()) return; + let refreshed: boolean; + try { refreshed = await options.refresh(); } + catch (error) { + const failure = options.mapRefreshError?.(error); + if (failure !== undefined) return failure; + throw error; + } + if (!refreshed) return; moves.set(options.requestKey, count + 1); options.setPermit(hop.permit); return await options.refetch(); diff --git a/src/server/responses/adapter-continuation.ts b/src/server/responses/adapter-continuation.ts index 086e0f7df93..16276d30cc5 100644 --- a/src/server/responses/adapter-continuation.ts +++ b/src/server/responses/adapter-continuation.ts @@ -1,5 +1,7 @@ import { carryProxyError, proxyErrorPresentation, isProxyError, rememberProxyErrorMessage } from "../../lib/client-error-body"; import { bindAnthropicRejectionResponse } from "../../adapters/anthropic-rejection-memo"; +import { deferPhysicalSendReporter } from "./deferred-send-reporter"; +import { responsesSpendTarget } from "./request-spend-target"; import { rotateAnthropicAccountOnResponseForInstance } from "../../oauth/anthropic-account-refusal"; import type { ResponsesRequestContext } from "./core-options"; import type { PreparedResponsesRequest } from "./request-prepare"; @@ -32,7 +34,7 @@ import { applyUpstreamRecoveryInit, isNonReplayableResponse, isSameTargetRetryHold, prepareSameTarget429Wait, - SendBudgetExhaustedError, + SendBudgetExhaustedError, errorDestinationLabel, } from "../../lib/upstream-retry"; import { unwrapUpstreamRetryEvidenceError } from "../../codex/auth-context"; import { AdmissionModelDeniedError, MODEL_NOT_ALLOWED_FOR_KEY } from "../admission-model-scope"; @@ -275,11 +277,13 @@ export function createAdapterContinuations( const fetchContinuationWithRetryPolicy = (route.provider.adapter === "google" || continuationTransientPolicy) ? fetchWithTransientRetry : fetchWithResetRetry; + const continuationReporter = transientSendReporter(); + const deferredContinuation = (continuationTransientPolicy || sendBudgetState.adapterSendBudget?.spendEnforced) + && route.providerName === "github-copilot" && route.provider.authMode !== "forward" + ? deferPhysicalSendReporter(continuationReporter) : undefined; return await fetchContinuationWithRetryPolicy( recovery => { - sendBudgetState.adapterSendBudget?.startRequest?.({ - poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, - }); + sendBudgetState.adapterSendBudget?.startRequest?.(responsesSpendTarget(logCtx, route.providerName)); transportState.noteRoutedAttemptSend(continuationEstimate, recovery ?? replayKind); return fetchWithHeaderTimeout( builtContinuationRequest.url, @@ -293,6 +297,9 @@ export function createAdapterContinuations( nextParsed.stream, providerFetch(route.provider, options.codexWsRuntimeIdentity, { dispatchOverride: oauthDispatch(builtContinuationRequest, nextParsed), + onPhysicalDispatch: deferredContinuation ? () => { + if (!deferredContinuation.beforeDispatch()) throw new SendBudgetExhaustedError(errorDestinationLabel(builtContinuationRequest.url)); + } : undefined, providerName: route.providerName, modelId: nextParsed.modelId, }), @@ -307,7 +314,7 @@ export function createAdapterContinuations( ...(continuationTransientPolicy || sendBudgetState.adapterSendBudget?.spendEnforced ? { attempts: remainingTransientSendBudget(continuationTransientPolicy?.attempts ?? 1), - onSendsConsumed: transientSendReporter(), + onSendsConsumed: deferredContinuation?.report ?? continuationReporter, } : {}), }, diff --git a/src/server/responses/adapter-dispatch.ts b/src/server/responses/adapter-dispatch.ts index 59e048c0d38..dd3d50c46ed 100644 --- a/src/server/responses/adapter-dispatch.ts +++ b/src/server/responses/adapter-dispatch.ts @@ -1,8 +1,9 @@ import { clientFacingModelText } from "../../lib/error-destination"; +import { responsesSpendTarget } from "./request-spend-target"; import { projectLocalModelRefusalLog } from "../request-log-failure-detail"; import { markProxyError, carryProxyError } from "../../lib/client-error-body"; import { codexAdapterBuildRefusal } from "./passthrough-build-error"; -import { recoverAutoPausedAccount } from "./account-auto-pause-recovery"; +import { createAutoPauseRefreshErrorMapper, recoverAutoPausedAccount } from "./account-auto-pause-recovery"; import { wasAccountAutoPaused, wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; import { bindAnthropicRejectionResponse } from "../../adapters/anthropic-rejection-memo"; import { deferPhysicalSendReporter } from "./deferred-send-reporter"; @@ -37,6 +38,7 @@ import { anthropicPoolCooldownResponse, clientCancelledResponse, readDisplaySafe import { isTranslatorBudgetExceededError } from "../../lib/translator-budget"; import { redactSecretString } from "../../lib/redact"; import { rewriteUpstreamPolicyRefusal } from "./policy-refusal"; +import { carryPolicyCandidateRefusal } from "./policy-request-scope"; import { withProviderRequestSlot } from "../../providers/request-pacing"; import { providerFetch, fetchWithHeaderTimeout, safeHostLabel } from "./fetch-helpers"; import { isAnthropicBillingKeyPool, rotateAnthropicBillingKeyOnResponse } from "../../providers/anthropic-billing-recovery"; @@ -171,7 +173,7 @@ export async function prepareAdapterExchange( | "genericFailovers" | "genericFailoverLimit" | "applyFailoverSnapshot" - | "noteRoutedAttemptSend" + | "noteRoutedAttemptSend" | "attemptSendDeferred" | "deferPhysicalAttribution" >, responseEffects: Pick, sendBudgetState: Pick< @@ -246,12 +248,11 @@ export async function prepareAdapterExchange( "pre-header", "connection-reset", ambiguousResendAllowanceFor(route.provider, requestIsSelfContained, claimAmbiguousResend), ).allowed; - const upstream = new AbortController(); const cleanupUpstreamAbort = linkAbortSignal(upstream, options.abortSignal); /** Contain replacement negotiation before adapter admission and release its upstream abort link. */ - const resolveRotatedCopilotSelection = async (): Promise => { - try { await resolveCopilotSelection(parsed); } + const resolveRotatedCopilotSelection = async (provider = route.provider): Promise => { + try { await resolveCopilotSelection(parsed, false, provider); } catch (error) { cleanupUpstreamAbort(); upstream.abort(); @@ -270,7 +271,6 @@ export async function prepareAdapterExchange( const antigravityPoolActivated = route.providerName === "google-antigravity" && isGenericOAuthFailoverEnabled(config, route.providerName); const antigravityDenial = createAntigravityModelDenialRecovery(); - // One immutable, body-safe outbound request per same-target sequence (URL, serialized body, // auth headers, generated compat headers). Same-target 429 replays reuse it verbatim; the // builder runs again only after a key/account/adapter rotation, an oauth refresh, or an @@ -479,14 +479,12 @@ export async function prepareAdapterExchange( ? fetchWithTransientRetry : fetchWithResetRetry; const initialReporter = transientSendReporter(compactPrepaid ?? options.comboDispatchPermit); - const deferredInitial = billingPoolRequest && !receiptMode ? deferPhysicalSendReporter(initialReporter) : undefined; + const deferredInitial = (billingPoolRequest || route.providerName === "github-copilot" && route.provider.authMode !== "forward") && !receiptMode ? deferPhysicalSendReporter(initialReporter) : undefined; try { upstreamResponse = await fetchWithRetryPolicy( recovery => { // Unconfigured generic sends may omit the counting reporter. Capture policy // independently so a later recovery still belongs to this request's start. - sendBudgetState.adapterSendBudget?.startRequest?.({ - poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, - }); + sendBudgetState.adapterSendBudget?.startRequest?.(responsesSpendTarget(logCtx, route.providerName)); // Receipt mode books the physical send at executor admission instead of here. if (!deferredInitial && !receiptMode && compactPrepaid && !compactPrepaidUsed) { if (!compactPrepaid.use()) throw new SendBudgetExhaustedError(errorDestinationLabel(builtInitialRequest.url)); @@ -699,8 +697,10 @@ export async function prepareAdapterExchange( ? retryRequest.usageLog.inputTokens : undefined; if (retryEstimate !== undefined) logCtx.usageLogInputTokens = retryEstimate; - logCtx.providerAdapter = transportState.activeAdapter.name; - sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); + if (!transportState.deferPhysicalAttribution) { + logCtx.providerAdapter = transportState.activeAdapter.name; + sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); + } recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); try { try { @@ -767,15 +767,13 @@ export async function prepareAdapterExchange( : undefined; let refetchPermitUsed = false; const refetchReporter = refetchAllowance ? transientSendReporter(refetchAllowance.permit) : undefined; - const deferredRefetch = (billingPoolRequest || options.comboAttempt) && refetchReporter + const deferredRefetch = (billingPoolRequest || options.comboAttempt || route.providerName === "github-copilot" && route.provider.authMode !== "forward") && refetchReporter ? deferPhysicalSendReporter(refetchReporter) : undefined; - const pendingKeyMetadata = deferredRefetch && usesApiKeyAccount(route.provider); + const pendingKeyMetadata = deferredRefetch && transportState.attemptSendDeferred(); try { return await refetchWithPolicy( recoveryKind => { - sendBudgetState.adapterSendBudget?.startRequest?.({ - poolId: logCtx.spendPoolId ?? route.providerName, identityId: logCtx.accountLogLabel, - }); + sendBudgetState.adapterSendBudget?.startRequest?.(responsesSpendTarget(logCtx, route.providerName)); // Prepare key metadata before the guarded transport commits its attempt; // counters and permits still wait for the physical executor after pacing. if (pendingKeyMetadata) transportState.noteRoutedAttemptSend(retryEstimate, recoveryKind ?? recovery); @@ -932,6 +930,7 @@ export async function prepareAdapterExchange( if (wasAccountAutoPaused(upstreamResponse) && route.provider.authMode !== "forward") { const recovered = await recoverAutoPausedAccount({ requestKey: transportState, reserve: reserveCredentialHop, remaining: transientSendCapFor(transientRetryPolicyFor(route.provider)?.attempts, sendBudgetState.targetSendsUsed), + mapRefreshError: createAutoPauseRefreshErrorMapper(upstreamResponse, upstream, cleanupUpstreamAbort, copilotRefusalResponse, () => (options.abortSignal ?? req.signal).aborted, clientCancelledResponse), refresh: () => transportState.recoverAutoPausedSelection(parsed), refetch: () => rebuildAndRefetch("oauth-account-429"), setPermit: permit => { sendBudgetState.pendingHopPermit = permit; } }); if (!recovered) break; if ("failed" in recovered) return recovered.failed; @@ -1035,11 +1034,6 @@ export async function prepareAdapterExchange( cleanupUpstreamAbort(); return markProxyError(formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(new Error("Cloud Code Assist project is required")))); } - transportState.sentOAuthSnapshot = refreshed; - transportState.replayOAuthCredentialSnapshot = { - accountId: refreshed.accountId, - generation: refreshed.generation, - }; if (route.providerName === "kiro") { parsed._kiroAuthContext = { ...(refreshed.kiro ?? {}) }; } @@ -1055,10 +1049,13 @@ export async function prepareAdapterExchange( ? resolveCopilotApiBaseUrl(refreshed.apiBaseUrl) : undefined, ); - route.provider = refreshedProvider; - const copilotRefusal = await resolveRotatedCopilotSelection(); + const copilotRefusal = await resolveRotatedCopilotSelection(refreshedProvider); if (copilotRefusal) return copilotRefusal; refreshedProvider = route.provider; + if (route.providerName !== "github-copilot") { + transportState.sentOAuthSnapshot = refreshed; + transportState.replayOAuthCredentialSnapshot = { accountId: refreshed.accountId, generation: refreshed.generation }; + } invalidateSameTargetRequest(); transportState.activeAdapter = resolveSelectionAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, refreshedProvider, inboundWire, route.staticPolicy), @@ -1508,7 +1505,7 @@ export async function prepareAdapterExchange( adapterName: transportState.activeAdapter.name, oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot, }); - sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); + if (!transportState.deferPhysicalAttribution) sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); // The replay IS this hop's send, so hand the reservation down and let the layer that // dispatches settle it: `adapterDispatchBudget` spends it on the adapter's first @@ -1613,7 +1610,7 @@ export async function prepareAdapterExchange( adapterName: transportState.activeAdapter.name, oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot, }); - sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); + if (!transportState.deferPhysicalAttribution) sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, transportState.activeAdapter.name, logCtx.accountLogLabel); recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, transportState.activeAdapter.name); // The replay IS this hop's send, so hand the reservation down and let the layer that // dispatches settle it: `adapterDispatchBudget` spends it on the adapter's first @@ -1962,7 +1959,7 @@ export async function prepareAdapterExchange( upstreamRetryAfter, includeDefault: !(transportState.anthropicPoolAccountId && classifyAnthropic429(upstreamResponse.headers) === "request-scoped-unknown"), }); - return formatUpstreamFailureResponse(upstreamResponse, normalized.code, + return carryPolicyCandidateRefusal(upstreamResponse, formatUpstreamFailureResponse(upstreamResponse, normalized.code, upstreamResponse.status, normalized.cyberPolicy ? (normalized.type ?? CYBER_POLICY_ERROR_CODE) : "upstream_error", message, @@ -1970,7 +1967,7 @@ export async function prepareAdapterExchange( ...(normalized.cyberPolicy ? { code: CYBER_POLICY_ERROR_CODE } : {}), ...(retryAfter !== undefined ? { retryAfter } : {}), }, - ); + )); } } diff --git a/src/server/responses/combo-required-tool-contract.ts b/src/server/responses/combo-required-tool-contract.ts index a5177b364d8..ccfe7356ec5 100644 --- a/src/server/responses/combo-required-tool-contract.ts +++ b/src/server/responses/combo-required-tool-contract.ts @@ -1,6 +1,7 @@ import { REQUIRED_TOOL_CONTRACT_CODE } from "../../lib/errors"; import { mapToolChoice } from "../../responses/parser-tools"; -import { modelInList, type OcxProviderConfig } from "../../types"; +import { extractHostedWebSearch } from "../../web-search/synthetic-tool"; +import { modelInList, type OcxProviderConfig, type OcxTool } from "../../types"; import { hasResponseMediaType } from "../../lib/http-response-semantics"; import { applyReplayRefusalClientHeaders, isNonReplayableResponse, markResponseNonReplayable } from "../../lib/upstream-retry"; import { clientWireOf } from "../inference/client-wire"; @@ -42,10 +43,28 @@ export function hasMandatoryToolChoice(rawBody: unknown): boolean { } /** A translated Chat target must not turn a mandatory selector into auto or none. */ -export function chatCanPreserveMandatoryToolChoice(rawBody: unknown, provider: OcxProviderConfig, modelId: string): boolean { +export function chatCanPreserveMandatoryToolChoice(rawBody: unknown, provider: OcxProviderConfig, modelId: string, effectiveTools?: readonly OcxTool[]): boolean { if (!hasMandatoryToolChoice(rawBody)) return true; if (modelInList(provider.autoToolChoiceOnlyModels, modelId)) return false; - const mapped = mapToolChoice((rawBody as Record).tool_choice); + const body = rawBody as Record; + const choice = body.tool_choice; + const tools = body.tools; + const hasDeclaredTools = Array.isArray(tools) && tools.length > 0 + || Array.isArray(body.input) && body.input.some(item => item && typeof item === "object" + && (item.type === "additional_tools" || item.type === "tool_search_output") + && Array.isArray(item.tools) && item.tools.length > 0); + // The parsed catalog includes additional_tools and tool_search_output. A static + // candidate check without it defers this decision to final request preparation. + // Hosted search belongs to the later sidecar injection owner. + if (choice === "required" && hasDeclaredTools && effectiveTools !== undefined && effectiveTools.length === 0 + && !extractHostedWebSearch(Array.isArray(tools) ? tools : undefined)) return false; + if (choice && typeof choice === "object" && !Array.isArray(choice)) { + const selector = choice as Record; + // Chat function names cannot preserve the selected MCP server's ownership. + if (selector.type === "allowed_tools" && Array.isArray(selector.tools) + && selector.tools.some(tool => tool && typeof tool === "object" && tool.type === "mcp")) return false; + } + const mapped = mapToolChoice(choice); return mapped === "required" || (!!mapped && typeof mapped === "object"); } diff --git a/src/server/responses/core-combo-failure.ts b/src/server/responses/core-combo-failure.ts index fb08a287591..8c90c487539 100644 --- a/src/server/responses/core-combo-failure.ts +++ b/src/server/responses/core-combo-failure.ts @@ -1,4 +1,5 @@ import { carryProxyErrorPresentation } from "../../lib/client-error-body"; +import { carryPolicyCandidateRefusal } from "./policy-request-scope"; import { comboFailureEvidence, carryProxyError } from "../../lib/client-error-body"; import { codexAccountModelRefusal, codexAccountModelRefusalHardStopCode, type CodexAccountModelRefusal } from "../../combos/failover"; import { parseRetryAfterMs } from "../../combos"; @@ -146,6 +147,7 @@ export async function consumeComboFailure( carryProxyError(response, failureResponse); carryReplayRefusal(response, failureResponse); carryProxyErrorPresentation(response, failureResponse); + carryPolicyCandidateRefusal(response, failureResponse); if (verifiedPlanRefusal) markVerifiedModelNotInPlanResponse(failureResponse); carryAnthropicRefusalCredential(response, failureResponse); carryCodexAccountCooldownResponse(response, failureResponse); diff --git a/src/server/responses/core-combo.ts b/src/server/responses/core-combo.ts index f073043989e..2f0db990694 100644 --- a/src/server/responses/core-combo.ts +++ b/src/server/responses/core-combo.ts @@ -7,6 +7,7 @@ import { effectiveAzureOutputBudget } from "../../adapters/azure"; import { buildResponsesTransport } from "../../adapters/openai-responses/transport"; import { isMetaAiResponsesDestination } from "../../responses/muse-tool-name-alias"; import { chatCanPreserveMandatoryToolChoice, hasMandatoryToolChoice } from "./combo-required-tool-contract"; +import { isPolicyCandidateRefusal, isMandatoryToolChoiceRefusal } from "./policy-request-scope"; import { isCodexAccountCooldownResponse } from "../../codex/account-cooldown-response"; import { resolveWireProtocolOverride } from "../adapter-resolve"; import { parseSyntheticRowId } from "../fast-row"; @@ -562,6 +563,8 @@ export async function executeComboResponses( const policy = captureRouteStaticPolicy(route.providerName, modelId, route.provider, route.staticPolicy.effectiveAlias, inboundWire); const provider = resolveWireProtocolOverride(route.providerName, modelId, route.provider, inboundWire, policy); + // Copilot's Auto account may settle on a different wire; the child checks that result. + if (route.providerName === "github-copilot" && provider.authMode !== "forward") return true; return provider.adapter === "openai-chat" ? chatCanPreserveMandatoryToolChoice(body, provider, modelId) : provider.adapter !== "openai-responses" || !isMetaAiResponsesDestination(buildResponsesTransport(provider, { headers: req.headers }).url); } catch { @@ -909,6 +912,7 @@ export async function executeComboResponses( // Local input admission refusals (#1524), while every failed target so far was one: such a // target sent nothing, so an exhausted ladder must not be reported as a provider rejection. let localAdmissionRefusals: string[] | undefined = []; + let toolChoiceRefusalsOnly: boolean | undefined; // The first quota/rate-limit refusal of the ladder. A fallback that then refuses its own // credential or plan (401/403, or a 400 the ladder hopped past) says nothing about why the // request failed: the primary target ran out. Returning that fallback refusal told clients @@ -919,7 +923,7 @@ export async function executeComboResponses( // Snapshotted before dispatch: a 401/403 inside this ladder cools its own provider, and that // fresh cooldown is not a reason the request found no target. const quotaCooldownSnapshot = snapshotComboQuotaCooldowns(config, comboId); - const exhaustedFailure = (): Response => { + const exhaustedFailure = (targetsExhausted = false): Response => { if (contextFailure) { adoptFailedChildLog(contextFailure.childLog); return contextFailure.response; @@ -927,6 +931,7 @@ export async function executeComboResponses( const returnsQuota = lastFailure && [400, 401, 403].includes(lastFailure.status) && quotaFailure; const failedChildLog = returnsQuota ? quotaFailedChildLog : lastFailedChildLog; if (failedChildLog) adoptFailedChildLog(failedChildLog); + if (targetsExhausted && toolChoiceRefusalsOnly === true) return comboUnavailableResponse("No available targets for combo; attempted targets cannot preserve mandatory tool_choice"); if (!lastFailure || ![400, 401, 403].includes(lastFailure.status)) return lastFailure!; if (quotaFailure) return quotaFailure; // The quota-refused target was cooled before this request picked, so the ladder never saw @@ -1376,6 +1381,7 @@ export async function executeComboResponses( } attemptRetained = true; lastFailure = failure.response; + toolChoiceRefusalsOnly = (toolChoiceRefusalsOnly ?? true) && isMandatoryToolChoiceRefusal(failure.response); // A real target failure ends the session pin; the next turn is decided afresh (#7525). jevPin?.forget(); if (!quotaFailure && (lastFailure.status === 429 || lastFailure.status === 402)) { @@ -1401,6 +1407,7 @@ export async function executeComboResponses( // send that already ran the turn, so no later target may receive it, whatever its status says. const failureDecision = failure.nonReplayable || spentReplacement ? "stop" + : isPolicyCandidateRefusal(failure.response) ? "hop" : comboFailureDecision(failure.response.status, failure.classificationText, { code: failure.upstreamCode, codexModelRefusal: failure.codexModelRefusal, @@ -1473,7 +1480,7 @@ export async function executeComboResponses( ); const failureNow = Date.now(); const attemptedTargets = pick.attempted; - const failureCooldownScope = comboFailureCooldownScope(failure.response.status, failure.classificationText, { + const failureCooldownScope = isPolicyCandidateRefusal(failure.response) ? "none" : comboFailureCooldownScope(failure.response.status, failure.classificationText, { code: failure.upstreamCode, }); const failedTargetKey = targetKey(pick.target); @@ -1584,7 +1591,7 @@ export async function executeComboResponses( if (!initialSend?.producerOwned) initialSend?.permit.release(); } } - const failure = exhaustedFailure(); + const failure = exhaustedFailure(true); if ( failure === lastFailure && lastFailure?.status === 413 && lastFailureClassifiesOverflow diff --git a/src/server/responses/core-lifetime.ts b/src/server/responses/core-lifetime.ts index 06e27a40c7b..14a24d51ee2 100644 --- a/src/server/responses/core-lifetime.ts +++ b/src/server/responses/core-lifetime.ts @@ -1,6 +1,7 @@ import { carryProxyError } from "../../lib/client-error-body"; import type { TranslatorBudget } from "../../lib/translator-budget"; import { carryResponseToolOwnership } from "./response-tool-ownership"; +import { carryPolicyCandidateRefusal } from "./policy-request-scope"; import { finalizeTranslatorBudgetResponse } from "../../lib/translator-budget"; import { carryReplayRefusal, isNonReplayableResponse, markResponseNonReplayable } from "../../lib/upstream-retry"; import { isVerifiedModelNotInPlanResponse, markVerifiedModelNotInPlanResponse } from "../../lib/errors"; @@ -57,6 +58,7 @@ export function finalizeOwnedTranslatorBudget(response: Response, budget: Transl if (isVerifiedModelNotInPlanResponse(response)) markVerifiedModelNotInPlanResponse(finalizedResponse); if (runTurnAdapterSseResponses.has(response)) runTurnAdapterSseResponses.add(finalizedResponse); carryCodexAccountCooldownResponse(response, finalizedResponse); + carryPolicyCandidateRefusal(response, finalizedResponse); return carryResponseToolOwnership(response, carryFormattedQuotaSource(response, finalizedResponse)); } @@ -86,6 +88,7 @@ export function finalizeAccountLease(response: Response, release: () => void, on if (runTurnAdapterSseResponses.has(response)) runTurnAdapterSseResponses.add(wrapped); // Retained compaction delivers through this lease wrapper; keep a local cooldown refusal local (#7850). carryCodexAccountCooldownResponse(response, wrapped); + carryPolicyCandidateRefusal(response, wrapped); return carryResponseToolOwnership(response, carryReplayRefusal(response, carryFormattedQuotaSource(response, wrapped))); } diff --git a/src/server/responses/deferred-send-reporter.ts b/src/server/responses/deferred-send-reporter.ts index f5463ab55cc..b4f33644354 100644 --- a/src/server/responses/deferred-send-reporter.ts +++ b/src/server/responses/deferred-send-reporter.ts @@ -1,4 +1,6 @@ -import type { PhysicalSendReporter } from "../../lib/request-execution-budget"; +import type { PhysicalSendReporter, SingleUseDispatchPermit } from "../../lib/request-execution-budget"; +import { SendBudgetExhaustedError } from "../../lib/upstream-retry"; +import type { ErrorDestinationLabel } from "../../lib/error-destination"; /** Retry thunks can fail in pacing; settle only starts admitted at the physical executor. */ export function deferPhysicalSendReporter(physical: PhysicalSendReporter) { @@ -23,3 +25,18 @@ export function deferPhysicalSendReporter(physical: PhysicalSendReporter) { }, }; } + +/** Keep legacy reporting unless a negotiated leg confirms its reservation at the physical send. */ +export function physicalSendAccounting(reporter: PhysicalSendReporter, deferred?: { + destination: () => ErrorDestinationLabel; permit?: SingleUseDispatchPermit; +}) { + const pending = deferred ? deferPhysicalSendReporter(reporter) : undefined; + let permitDispatched = false; + return { report: pending?.report ?? reporter, onPhysicalDispatch: pending ? () => { + if (permitDispatched) throw new SendBudgetExhaustedError(deferred!.destination()); + if (!pending.beforeDispatch() || deferred?.permit && !deferred.permit.use()) { + throw new SendBudgetExhaustedError(deferred!.destination()); + } + permitDispatched = Boolean(deferred?.permit); + } : undefined }; +} diff --git a/src/server/responses/fetch-helpers.ts b/src/server/responses/fetch-helpers.ts index 52d1db3d340..b754bd6ffed 100644 --- a/src/server/responses/fetch-helpers.ts +++ b/src/server/responses/fetch-helpers.ts @@ -58,9 +58,13 @@ export type AccountAutoPauseObserver = ((response: Response) => Promise void; }; /** Init fragment that reports the final send of the observer's attempt; empty when nothing observes. */ -export function physicalSendObserverInit(observer: AccountAutoPauseObserver | undefined, target: string): Record { +export function physicalSendObserverInit(observer: AccountAutoPauseObserver | undefined, target: string, onPhysicalSend?: () => void): Record { const physical = observer?.physical; - return physical ? { [PHYSICAL_SEND_OBSERVER]: (destination: string, headers: Headers) => physical(destination === target, headers) } : {}; + let noted = false; + return physical || onPhysicalSend ? { [PHYSICAL_SEND_OBSERVER]: (destination: string, headers: Headers) => { + if (onPhysicalSend && !noted) { onPhysicalSend(); noted = true; } + physical?.(destination === target, headers); + } } : {}; } // One WS attempt and its HTTP fallback are one logical physical-send receipt. const WS_FALLBACK_RECEIPT = Symbol.for("opencodex.ws-fallback-receipt"); diff --git a/src/server/responses/passthrough-delivery.ts b/src/server/responses/passthrough-delivery.ts index 04dd7c76a8a..0f4389eb516 100644 --- a/src/server/responses/passthrough-delivery.ts +++ b/src/server/responses/passthrough-delivery.ts @@ -1,5 +1,6 @@ import { clientFacingModelText } from "../../lib/error-destination"; import { projectLocalModelRefusalLog } from "../request-log-failure-detail"; +import { carryPolicyCandidateRefusal } from "./policy-request-scope"; import { markProxyError, rememberProxyErrorMessage, carryProxyError } from "../../lib/client-error-body"; import { releaseRequestOwnedMainOnOutcome, isKeyedRequestOwnedMain } from "../../codex/request-owned-affinity"; import { wasPermanentAccountRefusal } from "../../oauth/account-auto-pause"; @@ -662,13 +663,13 @@ export async function deliverPassthroughResponse( turnAdmissionLease: options.turnAdmissionLease, }); if (policyRefusal) return policyRefusal; - return carryProxyError(upstreamResponse, formatPassthroughUpstreamError(upstreamResponse.status, errorText, { + return carryPolicyCandidateRefusal(upstreamResponse, carryProxyError(upstreamResponse, formatPassthroughUpstreamError(upstreamResponse.status, errorText, { statusText: upstreamResponse.statusText, headers, // Provenance, not inference: `errorText` is empty when the bounded read finds nothing // display-safe, and an empty body is exactly what the retryable-429 default fires on. replayRefusal: isReplayRefusalResponse(upstreamResponse), - })); + }))); } if (options.nativeControl && isNativeControlResponse(upstreamResponse) && upstreamResponse.body) { diff --git a/src/server/responses/passthrough-dispatch.ts b/src/server/responses/passthrough-dispatch.ts index 4f1d1edee9a..814073d3485 100644 --- a/src/server/responses/passthrough-dispatch.ts +++ b/src/server/responses/passthrough-dispatch.ts @@ -3,6 +3,8 @@ import { markProxyError } from "../../lib/client-error-body"; import { CodexInstallationIdUnavailableError } from "../../codex/installation-id"; import { passthroughBuildErrorResponse } from "./passthrough-build-error"; import { recoverAutoPausedAccount, rotatePassthroughKeyOn429 } from "./passthrough-key-rotation"; +import { createAutoPauseRefreshErrorMapper } from "./account-auto-pause-recovery"; +import { physicalSendAccounting } from "./deferred-send-reporter"; import { adoptCodexWsStage, capturePassthroughAffinity } from "./passthrough-observations"; import { spendLedgerStorageErrorResponse } from "./spend-storage-error"; import { requestPacingOverloadResponse } from "./pacing-overload"; @@ -220,7 +222,7 @@ export async function preparePassthroughExchange( | "genericFailovers" | "genericFailoverLimit" | "applyFailoverSnapshot" - | "noteRoutedAttemptSend" + | "noteRoutedAttemptSend" | "deferPhysicalAttribution" | "selectionIsCurrent" | "requestBindings" >, @@ -800,13 +802,16 @@ export async function preparePassthroughExchange( * row replaces an HTTP stream, which a WS create frame is not, and the WebSocket row replaces * the transport that just failed. */ + const copilotSendAccounting = (permit?: Parameters[0]) => physicalSendAccounting( + transientSendReporter(permit), route.providerName === "github-copilot" && route.provider.authMode !== "forward" + ? { permit, destination: () => errorDestinationLabel(request.url) } : undefined); const sendAmbiguousReplacement = ( signal: AbortSignal = upstream.signal, ): Promise => { if (admissionState.authCtx.kind === "main" && admissionState.authCtx.mainQuotaDispatch) { admissionState.authCtx.mainQuotaDispatch = renewMainQuotaDispatchForAttempt(admissionState.authCtx.mainQuotaDispatch); } - const report = transientSendReporter(); + const accounting = copilotSendAccounting(), report = accounting.report; let started = false; const run = () => fetchWithHeaderTimeout( request.url, @@ -823,6 +828,7 @@ export async function preparePassthroughExchange( providerName: route.providerName, modelId: route.modelId, dispatchOverride: oauthDispatch(request, parsed, "passthrough"), + onPhysicalDispatch: accounting.onPhysicalDispatch ? () => { accounting.onPhysicalDispatch!(); started = true; } : undefined, beforeDispatch: headers => { if (signal.aborted) throw signal.reason; if (!transportState.selectionIsCurrent(transportState.requestBindings.get(request))) { @@ -843,8 +849,10 @@ export async function preparePassthroughExchange( // Charged to the SAME request counter every other send goes through. The // replacement is bought here rather than by a nested retry helper, so there is // one charge for one send and no per-layer counter to reconcile. - if (report.beforeSend?.() === false) throw new SendBudgetExhaustedError(errorDestinationLabel(request.url)); - started = true; + if (!accounting.onPhysicalDispatch) { + if (report.beforeSend?.() === false) throw new SendBudgetExhaustedError(errorDestinationLabel(request.url)); + started = true; + } }, }), route.provider.authMode === "forward", @@ -993,6 +1001,7 @@ export async function preparePassthroughExchange( // Stable per request: the Combo booking (`reserveDispatch`) already started and froze the spend // policy, so `spendEnforced` cannot change before dispatch even though no reporter starts it here. const receiptMode = !sendBudgetState.adapterSendBudget?.spendEnforced && Boolean(options.comboInitialSend); + const initialAccounting = copilotSendAccounting(); // #6969: Pool sends hold empty lifecycle frames so a pre-output refusal reaches the ladder below. const preOutputRefusals = new PoolPreOutputRefusals(); const preOutputRefusalHold = () => poolPreOutputRefusalHold(admissionState.authCtx, route.provider, connectMs, options.comboAttempt); @@ -1022,7 +1031,7 @@ export async function preparePassthroughExchange( dispatchOverride: oauthDispatch(request, parsed, "passthrough"), providerName: route.providerName, modelId: route.modelId, - onPhysicalDispatch: receiptMode ? sendBudgetState.noteInitialDispatch : undefined, + onPhysicalDispatch: receiptMode ? sendBudgetState.noteInitialDispatch : initialAccounting.onPhysicalDispatch, accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers), onCodexWsQuota: codexWsQuotaObserver(admissionState.authCtx, route.provider, route.modelId), beforeDispatch: isCanonicalOpenAiForwardProvider(route.provider) @@ -1038,7 +1047,7 @@ export async function preparePassthroughExchange( attempts: sendBudgetState.initialSendAllowance(transientSendAttempts()), // Reporter path: direct requests and enforced spend. Receipt-mode sends are settled // by `onPhysicalDispatch`; a reporter as well would count them twice. - ...(receiptMode ? {} : { onSendsConsumed: transientSendReporter() }), + ...(receiptMode ? {} : { onSendsConsumed: initialAccounting.report }), claimAmbiguousResend: claimPreHeaderResend, }, ); @@ -1062,13 +1071,10 @@ export async function preparePassthroughExchange( adapterName: refreshedAdapter.name, oauthCredentialSnapshot: transportState.replayOAuthCredentialSnapshot, }); - logCtx.providerAdapter = refreshedAdapter.name; - sealRequestAttemptIdentity( - logCtx.activeAttempt, - logCtx.provider, - refreshedAdapter.name, - logCtx.accountLogLabel, - ); + if (!transportState.deferPhysicalAttribution) { + logCtx.providerAdapter = refreshedAdapter.name; + sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, refreshedAdapter.name, logCtx.accountLogLabel); + } recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, refreshedAdapter.name); } /** Transfer adapter ownership without retaining the native abort link or request cache. */ @@ -1120,13 +1126,10 @@ export async function preparePassthroughExchange( : undefined; if (passthroughEstimate !== undefined) logCtx.usageLogInputTokens = passthroughEstimate; refreshUndeclaredToolGuard(request); - logCtx.providerAdapter = retryAdapter.name; - sealRequestAttemptIdentity( - logCtx.activeAttempt, - logCtx.provider, - retryAdapter.name, - logCtx.accountLogLabel, - ); + if (!transportState.deferPhysicalAttribution) { + logCtx.providerAdapter = retryAdapter.name; + sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, retryAdapter.name, logCtx.accountLogLabel); + } recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, route.provider, retryAdapter.name); const rebuiltBodyRefusal = refuseOversizedOutboundBody(request); if (rebuiltBodyRefusal) return { failed: rebuiltBodyRefusal }; @@ -1141,12 +1144,13 @@ export async function preparePassthroughExchange( { allowFinalRecoveryReserve: transientSendPolicy() === null }, ); if (recovery === "empty-completion") allowance.attempts = Math.min(1, allowance.attempts); + const recoveryAccounting = copilotSendAccounting(allowance.permit); try { return await fetchWithTransientRetry( innerRecovery => { // Gated on the return, not fire-and-forget: a consumed permit means this leg // already sent once, and letting the second call through would be a free send. - if (allowance.permit && !allowance.permit.use()) { + if (!recoveryAccounting.onPhysicalDispatch && allowance.permit && !allowance.permit.use()) { throw new SendBudgetExhaustedError(errorDestinationLabel(request.url)); } transportState.noteRoutedAttemptSend(passthroughEstimate, innerRecovery ?? recovery); @@ -1161,6 +1165,7 @@ export async function preparePassthroughExchange( && responseEffects.plaintextV2AgentMessageToolNames.size === 0 ? options.nativeControl : undefined, dispatchOverride: oauthDispatch(request, parsed, "passthrough"), + onPhysicalDispatch: recoveryAccounting.onPhysicalDispatch, providerName: route.providerName, modelId: route.modelId, accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers), @@ -1172,7 +1177,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), destination: errorDestinationLabel(request.url), attempts: allowance.attempts, - onSendsConsumed: transientSendReporter(allowance.permit), + onSendsConsumed: recoveryAccounting.report, claimAmbiguousResend: recovery === "empty-completion" ? undefined : claimPreHeaderResend }, ); } catch (err) { @@ -1190,6 +1195,7 @@ export async function preparePassthroughExchange( if (wasAccountAutoPaused(upstreamResponse) && route.provider.authMode !== "forward") { const recovered = await recoverAutoPausedAccount({ requestKey: transportState, reserve: reserveCredentialHop, remaining: transientSendCapFor(transientRetryPolicyFor(route.provider)?.attempts, sendBudgetState.targetSendsUsed), + mapRefreshError: createAutoPauseRefreshErrorMapper(upstreamResponse, upstream, cleanupUpstreamAbort, copilotRefusalResponse, () => (options.abortSignal ?? req.signal).aborted, clientCancelledResponse), refresh: () => transportState.recoverAutoPausedSelection(parsed), refetch: () => rebuildAndRefetch("oauth-account-429"), setPermit: permit => { sendBudgetState.pendingHopPermit = permit; } }); if (!recovered) break; if ("failed" in recovered) return recovered.failed; @@ -1358,11 +1364,6 @@ export async function preparePassthroughExchange( releaseCodexAuthContextProbeLease(admissionState.authCtx); return markProxyError(formatErrorResponse(401, "authentication_error", publicOAuthAuthenticationErrorMessage(new Error("Cloud Code Assist project is required")))); } - transportState.sentOAuthSnapshot = refreshed; - transportState.replayOAuthCredentialSnapshot = { - accountId: refreshed.accountId, - generation: refreshed.generation, - }; if (route.providerName === "kiro") { parsed._kiroAuthContext = { ...(refreshed.kiro ?? {}) }; } @@ -1378,10 +1379,13 @@ export async function preparePassthroughExchange( ? resolveCopilotApiBaseUrl(refreshed.apiBaseUrl) : undefined, ); - route.provider = refreshedProvider; - try { await resolveCopilotSelection(parsed); } + try { await resolveCopilotSelection(parsed, false, refreshedProvider); } catch (err) { return transportFailureResponse(err); } refreshedProvider = route.provider; + if (route.providerName !== "github-copilot") { + transportState.sentOAuthSnapshot = refreshed; + transportState.replayOAuthCredentialSnapshot = { accountId: refreshed.accountId, generation: refreshed.generation }; + } const refreshedAdapter = resolveSelectionAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, refreshedProvider, inboundWire, route.staticPolicy), config.cacheRetention, @@ -1413,6 +1417,7 @@ export async function preparePassthroughExchange( refreshUndeclaredToolGuard(request); const refreshedBodyRefusal = refuseOversizedOutboundBody(request); if (refreshedBodyRefusal) return refreshedBodyRefusal; + const oauthReplayAccounting = copilotSendAccounting(); try { upstreamResponse = await fetchWithTransientRetry( recovery => { @@ -1428,6 +1433,7 @@ export async function preparePassthroughExchange( && responseEffects.plaintextV2AgentMessageToolNames.size === 0 ? options.nativeControl : undefined, dispatchOverride: oauthDispatch(request, parsed, "passthrough"), + onPhysicalDispatch: oauthReplayAccounting.onPhysicalDispatch, providerName: route.providerName, modelId: route.modelId, accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers), @@ -1439,7 +1445,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), destination: errorDestinationLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: oauthReplayAccounting.report, claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); @@ -1558,6 +1564,7 @@ export async function preparePassthroughExchange( upstream.abort(); return clientCancelledResponse(); } + const failoverAccounting = copilotSendAccounting(); try { upstreamResponse = await fetchWithTransientRetry( recovery => { @@ -1575,6 +1582,7 @@ export async function preparePassthroughExchange( && responseEffects.plaintextV2AgentMessageToolNames.size === 0 ? options.nativeControl : undefined, dispatchOverride: oauthDispatch(request, parsed, "passthrough"), + onPhysicalDispatch: failoverAccounting.onPhysicalDispatch, providerName: route.providerName, modelId: route.modelId, accountAutoPause: (name, provider, headers) => captureAccountAutoPauseAttempt(name, provider, { config, codex: admissionState.authCtx }, headers), @@ -1586,7 +1594,7 @@ export async function preparePassthroughExchange( .then(adoptObservedResponse); }, { abortSignal: upstream.signal, label: safeHostLabel(request.url), destination: errorDestinationLabel(request.url), attempts: remainingTransientSendBudget(transientSendAttempts()), - onSendsConsumed: transientSendReporter(), claimAmbiguousResend: claimPreHeaderResend }, + onSendsConsumed: failoverAccounting.report, claimAmbiguousResend: claimPreHeaderResend }, ); } catch (err) { return transportFailureResponse(err); diff --git a/src/server/responses/policy-fallback.ts b/src/server/responses/policy-fallback.ts index bceb5aec1c6..7db10d29f0f 100644 --- a/src/server/responses/policy-fallback.ts +++ b/src/server/responses/policy-fallback.ts @@ -1,6 +1,7 @@ import { comboFailureDecision } from "../../combos/failover"; import { readBoundedResponseBody } from "../../lib/bounded-body"; import { isVerifiedModelNotInPlanResponse } from "../../lib/errors"; +import { isShadowSourceModel } from "../../lib/shadow-call"; import { isNonReplayableResponse } from "../../lib/upstream-retry"; import { finishRequestAttempt, type RequestLogContext } from "../request-log"; import { linkRequestSessionLane } from "../request-log-conversation"; @@ -13,7 +14,7 @@ import { captureCallerDirectAuth } from "../../providers/caller-authorization"; import { resolvePolicyProfileId } from "../../routing/profile"; import { parseSyntheticRowId } from "../fast-row"; import { routeConcreteModel } from "../../router"; -import { isPolicyCandidateRefusal, policyWireDestination, type PolicyRequestScope } from "./policy-request-scope"; +import { isPolicyCandidateRefusal, isMandatoryToolChoiceRefusal, mandatoryToolChoiceUnavailableResponse, policyWireDestination, type PolicyRequestScope } from "./policy-request-scope"; import { fallbackTriggerForStatus, guardFallbackHop } from "../../routing/fallback-policy"; import { captureAdmissionPolicy } from "../capture-admission-policy"; import { upstreamCodeForComboFailure } from "../../bridge/errors"; @@ -205,7 +206,15 @@ export async function handleResponsesWithPolicyFallback( && typeof (body as { model?: unknown }).model === "string") { const model = (body as { model: string }).model; const { fastRow, effortRow } = parseSyntheticRowId(model, config); - if (resolvePolicyProfileId(config, fastRow?.baseId ?? effortRow?.baseId ?? model) === null) return; + // A shadow-intercepted request is rewritten to its configured target after this hook — + // and that target may resolve through a routing profile. The snapshot has to exist now, + // while the body still names the source: a captured policy route without a replay body + // skips traversal and leaks the internal candidate refusal to the caller. + const intercept = config.shadowCallIntercept; + const shadowRewritable = Boolean(intercept?.enabled && intercept.model) + && isShadowSourceModel(model, intercept?.sourceModels); + if (resolvePolicyProfileId(config, fastRow?.baseId ?? effortRow?.baseId ?? model) === null + && !shadowRewritable) return; // Recovery and other core preparation may mutate the parsed body in place. Keep an // immutable snapshot of the original wire body so a retry cannot serialize those // mutations. Object-identity metadata is re-established by each attempt, not serialized. @@ -247,12 +256,17 @@ export async function handleResponsesWithPolicyFallback( })); let current = resolvedPolicyTarget(config, initialTrace.selected); const policyRoute = `policy/${initialTrace.profile!.id}`; + let toolChoiceRefusalsOnly = isMandatoryToolChoiceRefusal(response); while (!storedPool401ReplayDispatched && await shouldHopPolicyCandidate(response, req.signal)) { if (req.signal.aborted) return response; const next = rankPolicyFallbackCandidates(initialTrace, tried) .find(candidate => staysWithinPolicyAfterRedirect(config, policyEligibility, candidate, policyScope.triedDestinations)); - if (!next) return response; + if (!next) { + if (!toolChoiceRefusalsOnly) return response; + try { await response.body?.cancel(); } catch { /* the terminal diagnostic replaces this body */ } + return mandatoryToolChoiceUnavailableResponse(); + } // OCX-11: the next candidate is a hop; the fallback policy decides before it is sent, on the // redirected destination the guard above already validated. const nextTarget = resolvedPolicyTarget(config, next); @@ -281,16 +295,29 @@ export async function handleResponsesWithPolicyFallback( delete policyScope.preparedDestination; try { try { - const nextResponse = await runCore(retryRequest, config, logCtx, { ...coreOptions, policyFallbackCandidate: next }); + const nextResponse = await runCore(retryRequest, config, logCtx, { + ...coreOptions, + policyFallbackCandidate: next, + // A shadow-intercepted request retried on a candidate keeps helper isolation and + // its suppressed selector label; a first-party policy request leaves the flag unset. + ...(logCtx.shadowCallRewrittenFrom ? { shadowCallIntercepted: true } : {}), + }); + toolChoiceRefusalsOnly &&= isMandatoryToolChoiceRefusal(nextResponse); // A locally skipped route owns neither the returned failure nor its usage/settlement. // Preparation can replace route metadata and the tracker, or add fields absent before it. - if (isPolicyCandidateRefusal(nextResponse)) { + // The sendCount seam separates that skip from a refusal minted only after a physical + // send — a re-negotiated Copilot destination refused inside 401 or account recovery. + // The send happened, so the hop it rode commits below and the sent destination becomes + // the next hop's source, which is what a provider-scoped fallback refusal keys on. + if (isPolicyCandidateRefusal(nextResponse) && (logCtx.activeAttempt?.sendCount ?? 0) === 0) { + try { await nextResponse.body?.cancel(); } catch { /* settle the skipped candidate's lifetime owners */ } for (const key of Object.keys(logCtx)) { if (!Object.hasOwn(failureLog, key)) Reflect.deleteProperty(logCtx, key); } Object.assign(logCtx, failureLog); } else { // Dispatched: only a candidate that was not locally skipped counts as a taken hop. + try { await response.body?.cancel(); } catch { /* the dispatched candidate replaces the retained failure */ } hop.commit(); response = nextResponse; current = nextTarget; diff --git a/src/server/responses/policy-request-scope.ts b/src/server/responses/policy-request-scope.ts index 3504fcd7419..7459230a630 100644 --- a/src/server/responses/policy-request-scope.ts +++ b/src/server/responses/policy-request-scope.ts @@ -1,4 +1,5 @@ import { formatErrorResponse } from "../../bridge"; +import { markProxyError } from "../../lib/client-error-body"; import type { RouteResult } from "../../router"; import type { RouteDecisionTraceV1 } from "../../routing/trace"; import { resolveOpenAiVirtualModel } from "../../providers/openai-virtual-models"; @@ -27,12 +28,26 @@ export class PolicyCandidateUnavailableError extends Error { } } +export class MandatoryToolChoiceUnavailableError extends PolicyCandidateUnavailableError { + constructor() { + super(); + this.name = "MandatoryToolChoiceUnavailableError"; + } +} + +export function mandatoryToolChoiceUnavailableResponse(): Response { + return markProxyError(formatErrorResponse(400, "invalid_request_error", + "This mandatory tool_choice cannot be preserved for a translated Chat destination.")); +} + const localRefusals = new WeakSet(); +const mandatoryToolRefusals = new WeakSet(); export function policyCandidateRefusalResponse(error: unknown): Response | undefined { if (!(error instanceof PolicyCandidateUnavailableError)) return undefined; const response = formatErrorResponse(409, "policy_candidate_unavailable", error.message); localRefusals.add(response); + if (error instanceof MandatoryToolChoiceUnavailableError) mandatoryToolRefusals.add(response); return response; } @@ -41,6 +56,17 @@ export function isPolicyCandidateRefusal(response: Response): boolean { return localRefusals.has(response); } +export function isMandatoryToolChoiceRefusal(response: Response): boolean { + return mandatoryToolRefusals.has(response); +} + +/** Preserve local refusal authority when response lifetime owners wrap the body. */ +export function carryPolicyCandidateRefusal(source: Response, target: Response): Response { + if (localRefusals.has(source)) localRefusals.add(target); + if (mandatoryToolRefusals.has(source)) mandatoryToolRefusals.add(target); + return target; +} + export function capturePolicyRequestRoute(scope: PolicyRequestScope, route: RouteResult): RouteResult { if (!scope.eligibility && route.policyEligibility && route.routeDecision?.routeKind === "policy") { scope.eligibility = new Set(route.policyEligibility); diff --git a/src/server/responses/request-prepare.ts b/src/server/responses/request-prepare.ts index bf97d8f3aa2..ebda9c659ec 100644 --- a/src/server/responses/request-prepare.ts +++ b/src/server/responses/request-prepare.ts @@ -8,7 +8,7 @@ import type { PolicyRequestEvidence } from "../../routing/evaluator"; import type { RequestLogContext } from "../request-log"; import { markProtocolEntry, protocolMarkFeatures } from "../../protocols/trace"; import { isAntigravityRoute } from "../antigravity-direct-eligibility"; -import { capturePolicyRequestRoute, assertPolicyNormalizedRoute, recordPolicyPreparedDestination, policyCandidateRefusalResponse } from "./policy-request-scope"; +import { capturePolicyRequestRoute, assertPolicyNormalizedRoute, recordPolicyPreparedDestination, policyCandidateRefusalResponse, MandatoryToolChoiceUnavailableError, mandatoryToolChoiceUnavailableResponse } from "./policy-request-scope"; import { agentTaskRecoveryConfig, restoreCachedEncryptedAgentTasks, @@ -74,6 +74,7 @@ import { previousResponseProviderState, } from "../../responses/state"; import { formatErrorResponse } from "../../bridge"; +import { chatCanPreserveMandatoryToolChoice } from "./combo-required-tool-contract"; import { mapCodexAuthContextErrorToResponse } from "./codex-auth-error"; import type { OcxParsedRequest } from "../../types"; import { buildToolBridgeMaps } from "./collaboration"; @@ -1460,12 +1461,20 @@ export async function prepareResponsesRequest( }); try { assertPolicyNormalizedRoute(policyScope, admittedPolicyRoute, route); + const copilotNegotiationPending = route.providerName === "github-copilot" && route.provider.authMode !== "forward"; + if (!copilotNegotiationPending && route.provider.adapter === "openai-chat" + && !chatCanPreserveMandatoryToolChoice(parsed._rawBody, route.provider, route.modelId, parsed.context.tools ?? [])) { + // Combo children carry the refusal's local identity too, so a compatible sibling target + // can still be tried; a direct request keeps the client-facing 400 below. + if (policyScope.eligibility || options.comboAttempt) throw new MandatoryToolChoiceUnavailableError(); + return mandatoryToolChoiceUnavailableResponse(); + } } catch (error) { const policyRefusal = policyCandidateRefusalResponse(error); if (policyRefusal) return policyRefusal; throw error; } - // Normalization is the last thing that can move the destination: resolving an + // Static normalization can move the destination: resolving an // OpenAI virtual model rewrites route.modelId to the wire id that will // actually be billed. A scope checked only before this would authorize the // public selector and send the wire model, so the settled route is checked diff --git a/src/server/responses/request-send-budget.ts b/src/server/responses/request-send-budget.ts index 0417a6adcf5..f3528cc6d29 100644 --- a/src/server/responses/request-send-budget.ts +++ b/src/server/responses/request-send-budget.ts @@ -1,4 +1,5 @@ import { createInferenceSendBudget } from "../inference/context"; +import { responsesSpendTarget } from "./request-spend-target"; import type { ResponsesRequestContext } from "./core-options"; import { claimDispatchSpendProof, createPhysicalSendReporter, createRequestExecutionBudget, isRequestExecutionBudget, reportDispatchSends } from "../../lib/request-execution-budget"; import { @@ -84,8 +85,7 @@ export function createResponsesSendBudget( const noteTransientSends = (used: number): void => recordTransientSends(used, pendingHopPermit ?? inheritedPermit); // Capture at helper creation, before another leg can replace the pending handoff. const transientSendReporter = (permit = pendingHopPermit ?? inheritedPermit) => - createPhysicalSendReporter(sendBudget, () => ({ poolId: logCtx.spendPoolId ?? logCtx.provider, - identityId: logCtx.accountLogLabel }), permit, charged => { + createPhysicalSendReporter(sendBudget, () => responsesSpendTarget(logCtx), permit, charged => { noteTargetSends(Math.max(0, charged)); options.onCompactionRecoverySendsReported?.(charged); chargeWorkflowSends(workflowRootId, charged); diff --git a/src/server/responses/request-spend-target.ts b/src/server/responses/request-spend-target.ts new file mode 100644 index 00000000000..4ffcee38b86 --- /dev/null +++ b/src/server/responses/request-spend-target.ts @@ -0,0 +1,19 @@ +import type { SpendScopes } from "../../lib/spend-reservation-ledger"; +import type { RequestLogContext } from "../request-log"; + +type PhysicalSpendTarget = SpendScopes & { targetKey?: string }; +const targets = new WeakMap PhysicalSpendTarget>(); + +/** A prepared candidate can own admission without overwriting the last physical send's log. */ +export function bindResponsesSpendTarget(context: RequestLogContext, target?: () => PhysicalSpendTarget): void { + if (target) targets.set(context, target); + else targets.delete(context); +} + +/** Resolve late, after queued negotiation; ordinary callers retain their original pool fallback. */ +export function responsesSpendTarget(context: RequestLogContext, fallbackPool?: string): PhysicalSpendTarget { + return targets.get(context)?.() ?? { + poolId: context.spendPoolId ?? fallbackPool ?? context.provider, + identityId: context.accountLogLabel, + }; +} diff --git a/src/server/responses/request-transport.ts b/src/server/responses/request-transport.ts index e010fd9c4a1..ba867aa9917 100644 --- a/src/server/responses/request-transport.ts +++ b/src/server/responses/request-transport.ts @@ -65,10 +65,13 @@ import { resolveContextPrincipal } from "../auth-cors"; import { tryKiroAlternateAfterTerminalRefresh } from "../../oauth/kiro-terminal-failover"; import { classifyModelFamilyForQuota } from "../../oauth/account-quota-rank"; import { expandInferenceOAuthSendBudget } from "../inference/context"; -import { stampOAuthAccountLabel, usesApiKeyAccount } from "../../providers/label"; +import { stampOAuthAccountLabel, stampApiKeyAccountLabel, usesApiKeyAccount } from "../../providers/label"; +import { bindResponsesSpendTarget, responsesSpendTarget } from "./request-spend-target"; import { isXaiResponsesDestination, resolveProviderTransport } from "../../providers/xai-transport"; import { captureRouteStaticPolicy } from "../../router"; import { CopilotAutoHttpError, resolveCopilotAuto } from "../../providers/github-copilot-auto"; +import { PolicyCandidateUnavailableError, MandatoryToolChoiceUnavailableError, mandatoryToolChoiceUnavailableResponse, policyCandidateRefusalResponse } from "./policy-request-scope"; +import { chatCanPreserveMandatoryToolChoice } from "./combo-required-tool-contract"; import { AdmissionModelDeniedError, admissionModelDeniedResponse, projectAdmissionModelDeniedLog, @@ -140,6 +143,13 @@ export async function prepareResponsesTransport( ) { const { config, logCtx, options, req } = requestContext; const { route, parsed, inboundWire, translatorBudget } = requestState; + const deferPhysicalAttribution = route.providerName === "github-copilot" && route.provider.authMode !== "forward"; + let copilotCandidate: { model: string; adapter: string; accountLogLabel?: string } | undefined; + bindResponsesSpendTarget(logCtx); + if (deferPhysicalAttribution) bindResponsesSpendTarget(logCtx, () => ({ + poolId: logCtx.spendPoolId ?? route.providerName, + identityId: copilotCandidate ? copilotCandidate.accountLogLabel : logCtx.accountLogLabel, + })); const genericAffinityMode = genericSessionAffinityEnabled(config, route.providerName); const anthropicInstance = route.provider.authMode === "oauth" ? configuredAnthropicInstance(config, route.providerName) : undefined; @@ -334,8 +344,10 @@ export async function prepareResponsesTransport( options.accountLoad.lease?.release(); options.accountLoad.lease = replayLease; } - genericFailoverAccountId = admitted.accountId; - stampOAuthAccountLabel(logCtx, route.providerName, route.provider, admitted.accountId); + if (route.providerName !== "github-copilot") { + genericFailoverAccountId = admitted.accountId; + stampOAuthAccountLabel(logCtx, route.providerName, route.provider, admitted.accountId); + } return admitted; }; /** @@ -349,6 +361,10 @@ export async function prepareResponsesTransport( /** Project only bounded Copilot refusal metadata; upstream bodies and session tokens never reach the client. */ const copilotRefusalResponse = (error: unknown): Response | undefined => { if (error instanceof AdmissionModelDeniedError) return admissionModelDeniedResponse(projectAdmissionModelDeniedLog(error, logCtx)); + if (error instanceof PolicyCandidateUnavailableError) { + if (options.comboAttempt || options.policyRequestScope?.eligibility) return policyCandidateRefusalResponse(error); + return mandatoryToolChoiceUnavailableResponse(); + } if (!(error instanceof CopilotAutoHttpError) || ![401, 403, 429].includes(error.status)) return; const response = markProxyError(formatErrorResponse(error.status, error.status === 429 ? "rate_limit_error" : "authentication_error", error.message), "copilot"); @@ -379,20 +395,21 @@ export async function prepareResponsesTransport( return base; }; /** Resolve the request's ephemeral wire selection and recover at most one OAuth negotiation 401. */ - const resolveCopilotSelection = async (requestParsed: OcxParsedRequest, insideDispatch = false): Promise => { - if (route.providerName !== "github-copilot" || route.provider.authMode === "forward") return; + const resolveCopilotSelection = async (requestParsed: OcxParsedRequest, insideDispatch = false, candidateProvider: OcxProviderConfig = route.provider): Promise => { + if (route.providerName !== "github-copilot" || candidateProvider.authMode === "forward") { route.provider = candidateProvider; return; } + let provider = candidateProvider; copilotConfiguredWire ??= { - adapter: route.provider.adapter, - responsesPath: route.provider.responsesPath, - chatCompletionsPath: route.provider.chatCompletionsPath, - apiVersion: new Headers(route.provider.headers).get("X-GitHub-Api-Version") ?? undefined, + adapter: provider.adapter, + responsesPath: provider.responsesPath, + chatCompletionsPath: provider.chatCompletionsPath, + apiVersion: new Headers(provider.headers).get("X-GitHub-Api-Version") ?? undefined, }; /** Fence control-plane sends to the captured credential; reuse an already-held dispatch concurrency lease. */ const negotiate = () => { const binding: DispatchBinding | undefined = oauthSelection && servingOAuthSnapshot ? { kind: "oauth", selection: { ...oauthSelection }, snapshot: servingOAuthSnapshot } - : { kind: "api-key", provider: { ...route.provider } }; - return resolveCopilotAuto(copilotNegotiationBase(route.provider), copilotRequestedModel, requestParsed, + : { kind: "api-key", provider: { ...provider } }; + return resolveCopilotAuto(copilotNegotiationBase(provider), copilotRequestedModel, requestParsed, options.abortSignal ?? req.signal, /** Fence each negotiation send against the captured credential binding before touching the network. */ () => selectionIsCurrent(binding), !insideDispatch); }; @@ -400,14 +417,12 @@ export async function prepareResponsesTransport( try { selected = await negotiate(); } catch (error) { if (!(error instanceof CopilotAutoHttpError) || error.status !== 401 || copilotNegotiationRefreshed - || oauth401ReplayAttempted || route.provider.authMode !== "oauth" || !servingOAuthSnapshot) throw error; + || oauth401ReplayAttempted || provider.authMode !== "oauth" || !servingOAuthSnapshot) throw error; copilotNegotiationRefreshed = true; const refreshed = await refreshResolvedOAuthSelection(servingOAuthSnapshot); - route.provider = resolveProviderTransport(route.providerName, - { ...route.provider, apiKey: refreshed.accessToken }, parsed.options.promptCacheKey, + provider = resolveProviderTransport(route.providerName, + { ...provider, apiKey: refreshed.accessToken }, parsed.options.promptCacheKey, resolveCopilotApiBaseUrl(refreshed.apiBaseUrl)); - sentOAuthSnapshot = refreshed; - replayOAuthCredentialSnapshot = { accountId: refreshed.accountId, generation: refreshed.generation }; selected = await negotiate(); } assertRouteAllowedByScope( @@ -417,20 +432,40 @@ export async function prepareResponsesTransport( requestState.inboundSelector, { providerName: route.providerName, modelId: selected.modelId }, ); + const staticPolicy = captureRouteStaticPolicy(route.providerName, selected.modelId, selected.provider, + route.staticPolicy.effectiveAlias, inboundWire); + const autoAdapter = selected.auto ? selected.provider.adapter : undefined; + const negotiatedWire = autoAdapter === undefined + ? resolveWireProtocolOverride(route.providerName, selected.modelId, selected.provider, inboundWire, staticPolicy) + : { ...selected.provider, adapter: autoAdapter }; + if (negotiatedWire.adapter === "openai-chat" + && !chatCanPreserveMandatoryToolChoice(requestParsed._rawBody, negotiatedWire, selected.modelId, requestParsed.context.tools ?? [])) { + throw new MandatoryToolChoiceUnavailableError(); + } + // Credential admission may persist a refresh or newer manual selection. Publish request-local + // routing and accepted attribution only after this candidate's wire and scope both pass. copilotSessionExpiresAt = selected.expiresAt ?? Infinity; route.provider = selected.provider; route.modelId = selected.modelId; - route.staticPolicy = captureRouteStaticPolicy(route.providerName, route.modelId, route.provider, - route.staticPolicy.effectiveAlias, inboundWire); - copilotAutoAdapter = selected.auto ? selected.provider.adapter : undefined; + route.staticPolicy = staticPolicy; + copilotAutoAdapter = autoAdapter; for (const target of new Set([parsed, requestParsed])) { target.modelId = selected.modelId; if (target._rawBody && typeof target._rawBody === "object") (target._rawBody as { model?: string }).model = selected.modelId; target.context = renameRoutedIdentityInContext(target.context, selected.modelId); } - logCtx.model = selected.modelId; - if (logCtx.activeAttempt) logCtx.activeAttempt.model = selected.modelId; + const identity: { accountLogLabel?: string } = {}; + stampApiKeyAccountLabel(identity, route.providerName, route.provider); + stampOAuthAccountLabel(identity, route.providerName, route.provider, servingOAuthSnapshot?.accountId); + copilotCandidate = { model: selected.modelId, adapter: negotiatedWire.adapter, ...identity }; + if (servingOAuthSnapshot) { + // Recovery controllers snapshot admitted credentials before dispatch. Physical log + // attribution remains deferred, but these legacy recovery bindings must be available. + genericFailoverAccountId = servingOAuthSnapshot.accountId; + sentOAuthSnapshot = servingOAuthSnapshot; + replayOAuthCredentialSnapshot = { accountId: servingOAuthSnapshot.accountId, generation: servingOAuthSnapshot.generation }; + } }; /** * Apply a rotated account's FULL credential snapshot to the live route (#2568d). @@ -493,8 +528,7 @@ export async function prepareResponsesTransport( ) as OcxProviderConfig; } if (snapshot.projectId) rotatedProvider = { ...rotatedProvider, project: snapshot.projectId }; - route.provider = rotatedProvider; - await resolveCopilotSelection(retryParsed, insideDispatch); + await resolveCopilotSelection(retryParsed, insideDispatch, rotatedProvider); // Negotiation itself may have recovered a 401 with a newer committed credential. if (route.providerName === "github-copilot" && servingOAuthSnapshot) snapshot = servingOAuthSnapshot; if (route.providerName === "kiro") { @@ -518,28 +552,40 @@ export async function prepareResponsesTransport( // Re-stamp: a request that rotated accounts must be attributed to the account that actually // served it. All three rotation sites funnel through here, so this is the only re-stamp // needed -- and putting it anywhere else would let one of the three drift. - stampOAuthAccountLabel(logCtx, route.providerName, route.provider, snapshot.accountId); + if (!deferPhysicalAttribution) stampOAuthAccountLabel(logCtx, route.providerName, route.provider, snapshot.accountId); if (anthropicInstance !== undefined) { anthropicPoolAccountId = snapshot.accountId; logCtx.provider = formatAnthropicProviderForLog(anthropicInstance!, snapshot.accountId, config); - } else { + } else if (!deferPhysicalAttribution) { genericFailoverAccountId = snapshot.accountId; } - sentOAuthSnapshot = snapshot; + if (!deferPhysicalAttribution) sentOAuthSnapshot = snapshot; replayOAuthCredentialSnapshot = { accountId: snapshot.accountId, generation: snapshot.generation }; return snapshot; }; - // Key sends may be rebuilt while queued. Keep metadata pending until the guarded - // physical dispatch binds it to the selection that actually reaches the upstream. + // Key and Copilot sends may be rebuilt or refused while queued. Keep metadata pending + // until the guarded dispatch binds it to the selection that reaches the upstream. + const attemptSendDeferred = (): boolean => usesApiKeyAccount(route.provider) + || (route.providerName === "github-copilot" && route.provider.authMode !== "forward"); let pendingKeySend: { estimate: number | undefined; recovery?: AttemptRecoveryKind } | undefined; const noteRoutedAttemptSend = (estimate: number | undefined, recovery?: AttemptRecoveryKind): void => { - if (usesApiKeyAccount(route.provider)) pendingKeySend = { estimate, recovery }; + if (attemptSendDeferred()) pendingKeySend = { estimate, recovery }; else noteProviderAttemptSend(logCtx, route.providerName, route.provider, estimate, recovery); }; const commitKeyAttemptSend = (): void => { - if (!usesApiKeyAccount(route.provider)) return; + if (!attemptSendDeferred()) return; + if (copilotCandidate) { + logCtx.model = copilotCandidate.model; + logCtx.providerAdapter = copilotCandidate.adapter; + if (copilotCandidate.accountLogLabel) logCtx.accountLogLabel = copilotCandidate.accountLogLabel; + else delete logCtx.accountLogLabel; + if (servingOAuthSnapshot) { + genericFailoverAccountId = servingOAuthSnapshot.accountId; + sentOAuthSnapshot = servingOAuthSnapshot; + } + } noteProviderAttemptSend(logCtx, route.providerName, route.provider, - pendingKeySend?.estimate ?? logCtx.usageLogInputTokens, pendingKeySend?.recovery); + pendingKeySend?.estimate ?? logCtx.usageLogInputTokens, pendingKeySend?.recovery, copilotCandidate); pendingKeySend = undefined; }; const bindKeyUsageFromBridge = (usage: OcxUsage | undefined): void => { @@ -739,8 +785,7 @@ export async function prepareResponsesTransport( if (fresh) current = selectProactiveApiKeyTransport(config, route.providerName, fresh) ?? fresh; } if (!current) throw new Error("API key selection is unavailable before dispatch"); - route.provider = current; - await resolveCopilotSelection(requestParsed, true); + await resolveCopilotSelection(requestParsed, true, current); } adapter = activeAdapter = runTurnAdapter = resolveSelectionAdapter( resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire, route.staticPolicy), @@ -886,8 +931,7 @@ export async function prepareResponsesTransport( let physicalOwner: AnthropicPhysicalSendOwnership | null = null; try { requireAnthropicAdmission(); - if (!rebindPhysicalSend(options.sendBudget, { poolId: logCtx.spendPoolId ?? route.providerName, - identityId: logCtx.accountLogLabel })) throw new SendBudgetExhaustedError(); + if (!rebindPhysicalSend(options.sendBudget, responsesSpendTarget(logCtx, route.providerName))) throw new SendBudgetExhaustedError(); if (snapshot) { // Reserve the incarnation at this synchronous send boundary, never on return. physicalOwner = captureAnthropicPhysicalSendOwnership(snapshot); @@ -897,7 +941,10 @@ export async function prepareResponsesTransport( sentOAuthSnapshot = snapshot; passiveQuotaWriterGeneration = writerGeneration; } - commitKeyAttemptSend(); + // Copilot's local negotiation may refuse before this point, and spend admission + // may still refuse at the HTTP executor. Its note follows that final admission. + const deferredCopilotNote = route.providerName === "github-copilot" && route.provider.authMode !== "forward"; + if (!deferredCopilotNote) commitKeyAttemptSend(); // The WS executor dials its own socket: HTTP connection policy and body coding do not apply. // WS eligibility (egress, pacing, URL) was checked for the provider it was built for; a // provider reselected above was not, so it takes the HTTP fallback under its own route. @@ -905,7 +952,7 @@ export async function prepareResponsesTransport( distributedKeyPhysicalFetch(anthropicBillingPhysicalFetch(fetchImpl, route.providerName, route.provider), route.providerName, route.provider, target), input, { ...physicalInit, redirect: "manual", - ...physicalSendObserverInit(observeAutoPause, destination instanceof Request ? destination.url : String(destination)) }, + ...physicalSendObserverInit(observeAutoPause, destination instanceof Request ? destination.url : String(destination), deferredCopilotNote ? commitKeyAttemptSend : undefined) }, { providerName: route.providerName, provider: route.provider }, )) as typeof globalThis.fetch; response = await fetchWithHeaderTimeout(target, dispatchInit, dispatchInit.signal ?? options.abortSignal ?? req.signal, @@ -1126,20 +1173,18 @@ export async function prepareResponsesTransport( } if (admitted.accountId !== resolved.accountId) usedPreferredAccount = true; resolved = admitted; - replayOAuthCredentialSnapshot = { - accountId: resolved.accountId, - generation: resolved.generation, - }; - if (isOAuth401ReplayProvider) sentOAuthSnapshot = resolved; route.provider = { ...route.provider, apiKey: resolved.accessToken }; - // Attribution is independent of failover (#2699): stamped from the resolved snapshot - // itself, not from inside the `isGenericFailoverProvider` branch below, so a future - // narrowing of that predicate cannot silently switch attribution off. - stampOAuthAccountLabel(logCtx, route.providerName, route.provider, resolved.accountId); + // Copilot publishes accepted attribution after negotiation. Other providers retain + // snapshot-based attribution independently of the failover predicate (#2699). + if (route.providerName !== "github-copilot") { + replayOAuthCredentialSnapshot = { accountId: resolved.accountId, generation: resolved.generation }; + if (isOAuth401ReplayProvider) sentOAuthSnapshot = resolved; + stampOAuthAccountLabel(logCtx, route.providerName, route.provider, resolved.accountId); + } // Remember which account actually served this request so a 429 cools THAT one, not // whichever account is active by the time the response comes back (#2568). if (isGenericFailoverProvider(route.providerName, route.provider)) { - genericFailoverAccountId = resolved.accountId; + if (route.providerName !== "github-copilot") genericFailoverAccountId = resolved.accountId; // Advance the pool cursor only now that this account is actually admitted. The // helper returns immediately unless the kernel is on AND the strategy is // round-robin, so quota and fill-first pools reach it without being touched. @@ -1218,7 +1263,7 @@ export async function prepareResponsesTransport( route.provider, parsed.options.promptCacheKey, route.providerName === "github-copilot" && route.provider.authMode === "oauth" - ? resolveCopilotApiBaseUrl(sentOAuthSnapshot?.apiBaseUrl) + ? resolveCopilotApiBaseUrl(servingOAuthSnapshot?.apiBaseUrl) : undefined, ); try { await resolveCopilotSelection(parsed); } @@ -1259,7 +1304,7 @@ export async function prepareResponsesTransport( forwardHeaders: requestState.selectedForwardHeaders, }); assignNativeLogConversationId(logCtx, normalizeLogConversationId(parsed._cursorConversationId)); - logCtx.providerAdapter = adapter.name; + if (!deferPhysicalAttribution) logCtx.providerAdapter = adapter.name; // Ordinary requests receive one durable attempt only after their final initial // adapter is resolved. Combo children own their attempt and retries keep it. if (!options.comboAttempt && !logCtx.activeAttempt) { @@ -1273,7 +1318,7 @@ export async function prepareResponsesTransport( logCtx.activeAttemptStartedAt = Date.now(); (logCtx.attempts ??= []).push(attempt); } - sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, adapter.name, logCtx.accountLogLabel); + if (!deferPhysicalAttribution) sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, adapter.name, logCtx.accountLogLabel); recordAttemptCredentialSource(logCtx.activeAttempt, route.providerName, adapterProvider, adapter.name); runTurnAdapter = adapter; if (adapter.runTurn) { @@ -1309,13 +1354,15 @@ export async function prepareResponsesTransport( // Freeze the request ceiling before any 429 writes cooldowns. Selection still reads // live eligibility on every hop; cooled accounts cannot shorten this request's allowance. // The snapshot is clamped: a larger roster must not raise one request's hops or sends. - const genericRosterSize = genericFailoverAccountId ? new Set([ - genericFailoverAccountId, + const fundingAccountId = deferPhysicalAttribution && isGenericFailoverProvider(route.providerName, route.provider) + ? servingOAuthSnapshot?.accountId : genericFailoverAccountId; + const genericRosterSize = fundingAccountId ? new Set([ + fundingAccountId, ...eligibleFailoverAccounts(route.providerName, Date.now(), classifyModelFamilyForQuota(route.providerName, route.modelId)), ]).size : 0; const fundedAccounts = Math.min(genericRosterSize, GENERIC_OAUTH_MAX_ACCOUNTS_PER_REQUEST); const genericFailoverLimit = Math.max(GENERIC_OAUTH_MAX_FAILOVERS_PER_REQUEST, fundedAccounts - 1); - if (genericFailoverAccountId) expandInferenceOAuthSendBudget(options.sendBudget, fundedAccounts); + if (fundingAccountId) expandInferenceOAuthSendBudget(options.sendBudget, fundedAccounts); return { genericFailoverLimit, @@ -1423,6 +1470,8 @@ export async function prepareResponsesTransport( refreshRunTurnAdapter, oauthDispatch, noteRoutedAttemptSend, + attemptSendDeferred, + deferPhysicalAttribution, commitKeyAttemptSend, recoverAutoPausedSelection, bindKeyUsageFromBridge, diff --git a/structure/dashboard-and-usage.md b/structure/dashboard-and-usage.md index 4b27eae72f7..7e1f3f7f9bf 100644 --- a/structure/dashboard-and-usage.md +++ b/structure/dashboard-and-usage.md @@ -180,7 +180,7 @@ identity; rotating the secret behind the same reference preserves the logical ac `src/usage/jev-stats.ts` owns the parallel content-free JEV projection; its accumulator contract lives in [providers/jev-decision.md](./providers/jev-decision.md). `src/providers/label.ts` stamps only key authentication, including implicit custom-provider -keys. `src/server/request-log.ts` commits identity at dispatch after queued selection changes, +keys. `src/server/request-log.ts` (with recovery markers in `src/server/request-log-attempt-recovery.ts`) commits identity at dispatch after queued selection changes, retains separate flat records when retries change keys, and isolates each record's raw usage from parent combo totals and adapter-loop aggregation. Reported failure usage is retained; missing usage and historical identities remain unknown. Native wire snapshots replace only the diff --git a/structure/transports/policy-fallback.md b/structure/transports/policy-fallback.md index 196d0c90a96..9c2c2239dc8 100644 --- a/structure/transports/policy-fallback.md +++ b/structure/transports/policy-fallback.md @@ -6,6 +6,12 @@ Subagent selection and encrypted-task recovery keep that membership; a replaceme refused before dispatch, and policy fallback may continue with another originally eligible candidate. A shadow-intercept target is probed without that capture and is captured only once interception is accepted, so a declined policy target never scopes the request's own route. +The outer wrapper snapshots the original wire body of any request whose model a configured shadow +intercept can rewrite: interception runs after that snapshot, so a routing-profile target selected +by interception keeps the same candidate traversal and exhausted-refusal 400 a direct policy +request has. A retried candidate replays the pre-interception body under the concrete candidate +selector and carries the interception's helper isolation and suppressed label through +`shadowCallIntercepted`. `src/server/responses/policy-fallback.ts` inspects and executes each retry as a concrete candidate, so a public combo or profile alias cannot reselect its destination. Redirects remain inside the original evaluation, including eligible destinations omitted from the bounded diagnostic trace. @@ -16,17 +22,42 @@ When a main-account hard-lock fallback selects a profile, `src/server/responses/ checks its first ranked destination against the original model under the [fallback policy](../fallback-policy.md#sites), before authentication or upstream dispatch. +After final normalization, preparation refuses a Chat destination that cannot preserve a mandatory +tool choice using `MandatoryToolChoiceUnavailableError`. The local response identity allows traversal +to a compatible originally eligible candidate without sending to the incompatible destination. +Ordinary direct routes keep the client-facing 400; upstream errors do not acquire this identity. +Non-forward Copilot routes defer this check until Auto negotiation settles the adapter and model. +`src/server/responses/request-transport.ts` repeats the check after every negotiation, including +credential and session renewal; a static Responses route cannot silently become incompatible Chat. +Copilot stages its provider/model until negotiation, scope and mandatory-tool checks pass. `src/server/responses/request-spend-target.ts` supplies the accepted candidate's identity to spend admission without overwriting physical attribution, and clears that binding for each new transport preparation. Only a physical send publishes the candidate's account/model/adapter to its log; account, model or wire changes seal the previous attempt before applying the new identity. Admitted roster identity still funds only ingress-owned budgets before the first send. Persisted refreshes, manual selection and paused-account fences remain authoritative. +Policy and combo children carry the local refusal identity; direct requests retain the same 400. +Direct and exhausted-policy tool refusals carry proxy-error provenance without a Copilot account-error presentation, so Chat and Messages retain the local diagnostic. +Auto-pause credential refresh maps the same typed refusal before returning from recovery, while its +unused hop reservation is released. Adapter and native Responses error delivery carry that identity +through response rebuilding so policy traversal also works after a mid-dispatch renegotiation. +Every refresh exception cancels its original response and releases the exchange's abort linkage; +caller cancellation uses the selected effective signal and is captured before cleanup aborts the upstream, so cleanup or an unused request signal cannot manufacture it. +Queued Copilot refusals consume no physical-send budget or attempt count. Initial, recovery and continuation reporters settle at the HTTP executor; the Copilot attempt note follows spend admission through the physical-send observer. Other OAuth runTurn notes and forward/Codex receipt paths keep their existing owners. Single-use recovery reservations reject a second dispatch before another count is recorded. + Normal virtual-model wire mapping remains attached to its admitted selector and does not add a new eligible model. Policy traversal records settled physical destinations and skips a destination already attempted through another candidate or redirect; same-target credential and transient retries retain their existing budgets. Local skips preserve the last upstream failure and its log context, including route identity, usage, active attempt and spend tracker; preparation-only fields are discarded. +A refusal minted only after a physical send — a re-negotiated destination refused inside +credential recovery — is a dispatched candidate instead: the hop it was sent on commits and +that sent destination becomes the next hop's source for the fallback guard. Completed physical attempt rows remain intact, and final logging settles the last actual send's usage. The initial decision remains the selection evidence while physical attempts record execution. +When every observed refusal is specifically a mandatory-tool incompatibility, exhausted policy traversal returns the direct 400 diagnostic. Generic local skips and upstream failures permanently exclude that projection; cancellation and replay boundaries retain precedence. Discarded local bodies and replaced upstream failure bodies are cancelled to settle their lifetime owners; a refused fallback guard also cancels the retained failure. Cancellation, non-replayable responses and committed output keep the [shared replay boundaries](responses-failover.md). A local policy refusal is identified by its in-process response identity; matching upstream prose or error codes cannot authorize another attempt. +The translator-budget and account-lease wrappers in `src/server/responses/core-lifetime.ts` carry +that identity from their source response without deriving it from serialized error contents. +`src/server/responses/core-combo-failure.ts` carries it when rebuilding a consumed failure, so the +combo can skip that local candidate while copied upstream bytes remain ordinary failures. Formatted quota errors retain [their upstream-code provenance](responses-combo-boundaries.md#upstream-account-quota-exhaustion-failover) through lifecycle wrappers; a reflected quota message on a 400/422 response does not authorize a candidate hop merely because formatting serialized it as `insufficient_quota` or its text mentions a free-tier single-request cap. @@ -35,3 +66,10 @@ Coverage: `tests/routing/routing-policy-dispatch-scope.test.ts` exercises real r normalization and configured subagent selection with synthetic dispatch. It also covers recovery, alias collisions, redirects, bounded traces and virtual models. `tests/routing/routing-policy-fallback.test.ts` pins request snapshots, terminal responses, cancellation, credential handoff and attempt accounting. +It also covers shadow-intercepted policy routes: candidate traversal, upstream failover, exhaustion +to the client-facing 400, and isolation carry onto the retried candidate. +`tests/responses/responses-direct-chat-tool-choice.test.ts` exercises the real endpoint's pre-send +Chat refusal, unchanged hosted/MCP selectors and tool declarations, and native MCP output on Responses fallback. +It also runs a shadow-intercepted request through the real endpoint to a compatible policy candidate. +`tests/adapters/translator-budget-lifetime.test.ts` distinguishes a local refusal from identical +upstream wire bytes through both lifetime wrappers followed by combo failure rebuilding, including the tool-specific identity and exactly-once release of leases and translator accounting. diff --git a/structure/transports/responses-combo-boundaries.md b/structure/transports/responses-combo-boundaries.md index d333e07a5c1..7563a852deb 100644 --- a/structure/transports/responses-combo-boundaries.md +++ b/structure/transports/responses-combo-boundaries.md @@ -37,8 +37,23 @@ The child parser accepts `tool_search`, `computer_use` and `image_gen` as recogn Its raw body preserves the selector; existing destination compatibility, including routed private tool-search lowering, still applies after parsing. Chat translation maps a direct `tool_search` choice to its existing synthetic function name. MCP null names also parse inside required allowed-tools lists; function/custom null names remain invalid. A Chat destination is skipped before send when translation would degrade a mandatory choice to auto/none, including the resolved model's auto-only tool-choice policy; exhaustion reports the tool-choice constraint. Eligibility resolves synthetic Fast/effort selectors before model policy, matching child request preparation while preserving exact configured IDs. +For `required` with nonempty declarations at the top level or in `additional_tools`/`tool_search_output`, final preparation and Copilot renegotiation check the full parsed tool union; an empty union refuses Chat unless top-level hosted search belongs to later sidecar injection. Static candidate checks defer this decision until parsed tools are available. Registered OpenAI API virtual models resolve to their wire model before adapter and auto-only policy checks, using the same registry resolver as child normalization. Other providers and unregistered IDs remain literal. Coverage: `tests/server/server-combo-required-tool-contract.test.ts`. +Required allowed-tools lists containing MCP entries also exclude Chat: a function name cannot retain +the MCP server's ownership, even when a client function has the same name. The shared predicate in +`src/server/responses/combo-required-tool-contract.ts` also guards direct Chat destinations after final +normalization in `src/server/responses/request-prepare.ts`. An incompatible direct route returns 400 +before dispatch; an authorized policy candidate uses the [local policy refusal](policy-fallback.md) +so another originally eligible destination can serve the unchanged choice. +Copilot targets that negotiate Auto defer the static eligibility verdict to the child's final wire +and model check. A locally minted policy-candidate refusal survives failure rebuilding and permits a +hop without inferring authority from its serialized code or text. Non-replayable responses and spent +replacement grants still stop before this local-skip branch. A local candidate refusal does not cool +the target, so later compatible requests can select the healthy target again. `tests/providers/github-copilot/github-copilot-auto-runtime.test.ts` +and `tests/adapters/translator-budget-lifetime.test.ts` cover negotiation and refusal provenance. +On target-ladder exhaustion, a private tool-specific refusal identity yields the existing 503 combo-unavailable diagnostic only while every observed failure has that identity. Any generic local or upstream failure keeps the ordinary exhaustion result; copied error bytes cannot acquire the identity. A denied later send reservation keeps the retained failure instead of projecting this diagnostic. The diagnostic describes attempted targets, without claiming that eligibility-skipped targets were tried. + Before combo success bookkeeping, native delivery in `src/server/responses/passthrough-delivery.ts` uses `src/server/responses/combo-required-tool-contract.ts` to check successful native `openai-responses` JSON children with explicit required or named function/custom choices. diff --git a/tests/adapters/translator-budget-lifetime.test.ts b/tests/adapters/translator-budget-lifetime.test.ts index dc6de8e7c53..7dd1901ba9c 100644 --- a/tests/adapters/translator-budget-lifetime.test.ts +++ b/tests/adapters/translator-budget-lifetime.test.ts @@ -7,6 +7,11 @@ import { finalizeAccountLease, finalizeOwnedTranslatorBudget, runTurnAdapterSseResponses, } from "../../src/server/responses/core-lifetime"; import { isNonReplayableResponse, markResponseNonReplayable } from "../../src/lib/upstream-retry"; +import { + isPolicyCandidateRefusal, PolicyCandidateUnavailableError, policyCandidateRefusalResponse, + isMandatoryToolChoiceRefusal, MandatoryToolChoiceUnavailableError, +} from "../../src/server/responses/policy-request-scope"; +import { consumeComboFailure } from "../../src/server/responses/core-combo-failure"; import { markNativePassthroughSseResponse, isNativePassthroughSseResponse, markEagerRelaySseResponse, isEagerRelaySseResponse, @@ -176,6 +181,48 @@ for (const outerLease of [false, true]) { } } +test.each([true, false])("policy refusal authority survives lifetime wrappers only for a local response (%s)", async local => { + const refusal = policyCandidateRefusalResponse(new PolicyCandidateUnavailableError())!; + // An upstream can reproduce every wire byte, but cannot mint the local response identity. + const source = local ? refusal : new Response(await refusal.text(), { status: refusal.status, headers: refusal.headers }); + let released = 0; + const budget = createTranslatorBudget(); + const wrapped = finalizeAccountLease(finalizeOwnedTranslatorBudget(source, budget), () => { released++; }); + expect(isPolicyCandidateRefusal(wrapped)).toBe(local); + expect(wrapped.status).toBe(409); + expect((await wrapped.json()).error.code).toBe("policy_candidate_unavailable"); + expect(released).toBe(1); + expect(budget.snapshot().currentBytes).toBe(0); +}); + +for (const mode of ["read", "cancel"] as const) test.each(["generic", "tool", "spoof"])( + `combo failure after both lifetime wrappers retains authority and releases once (${mode}, %s)`, async kind => { + const baseline = translatorLiveBudgetCountForTests(), aggregate = translatorObservedBufferSnapshot(); + const refusal = policyCandidateRefusalResponse(kind === "generic" + ? new PolicyCandidateUnavailableError() : new MandatoryToolChoiceUnavailableError())!; + const source = kind !== "spoof" ? refusal : new Response(await refusal.text(), { status: refusal.status, headers: refusal.headers }); + let released = 0; + const budget = createTranslatorBudget(); + budget.openCall("refusal"); + budget.chargeRetained(512, { kind: "tool_args", callId: "refusal" }); + const wrapped = finalizeAccountLease(finalizeOwnedTranslatorBudget(source, budget), () => { released++; }); + let rebuilt: Response | undefined; + try { + rebuilt = (await consumeComboFailure(wrapped)).response; + expect(rebuilt.status).toBe(409); + expect(isPolicyCandidateRefusal(rebuilt)).toBe(kind !== "spoof"); + expect(isMandatoryToolChoiceRefusal(rebuilt)).toBe(kind === "tool"); + expect(source.bodyUsed).toBe(true); expect(wrapped.bodyUsed).toBe(true); + expect(released).toBe(1); + expect(budget.snapshot()).toMatchObject({ currentBytes: 0, activeCalls: 0 }); + expect(translatorLiveBudgetCountForTests()).toBe(baseline); + expect(translatorObservedBufferSnapshot()).toMatchObject({ currentBytes: aggregate.currentBytes, active: aggregate.active }); + if (mode === "read") expect((await rebuilt.json()).error.code).toBe("upstream_error"); + else await rebuilt.body?.cancel(); + expect(released).toBe(1); + } finally { await rebuilt?.body?.cancel().catch(() => {}); await wrapped.body?.cancel().catch(() => {}); budget.dispose(); } +}); + for (const surface of ["responses", "chat-completions", "claude-messages"] as const) { test(`${surface}: HTTP owner links request abort to its unread response budget`, async () => { const handle = surface === "responses" diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 270d82a7c2f..e2274b348c4 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -306,7 +306,8 @@ "openai-chat-tool-call-id-remint.test.ts": "adapters/openai", "coding-agent-json-lines-scaling.test.ts": "providers", "usage-snapshot-digest-reuse.test.ts": "usage", "worker-embed.test.ts": "lib", "worker-embed-coverage.test.ts": "lib", "jev-stats.test.ts": "usage", - "responses-devin-401-replay.test.ts": "responses", "responses-oauth-refresh-capacity.test.ts": "responses", "responses-grok-devin-preflight.test.ts": "responses", + "responses-devin-401-replay.test.ts": "responses", + "responses-direct-chat-tool-choice.test.ts": "responses", "responses-oauth-refresh-capacity.test.ts": "responses", "responses-grok-devin-preflight.test.ts": "responses", "abort-idle-deadline.test.ts": "lib", "tool-envelope-echo-whole-line.test.ts": "adapters", "abort-race.test.ts": "adapters", "account-import.test.ts": "server", "account-import-project-fallback.test.ts": "server", diff --git a/tests/helpers/anthropic-billing-send-budget-driver.ts b/tests/helpers/anthropic-billing-send-budget-driver.ts index eac6e301f2b..b57809b7b5e 100644 --- a/tests/helpers/anthropic-billing-send-budget-driver.ts +++ b/tests/helpers/anthropic-billing-send-budget-driver.ts @@ -106,6 +106,9 @@ async function probe(firstStatus: 401 | 429 | 400) { resolveSelectionAdapter: () => adapter, resolveCopilotSelection: async () => {}, noteRoutedAttemptSend: () => {}, + // Mirror key-account transport metadata deferral; this fixture's log callback is a no-op. + attemptSendDeferred: () => true, + deferPhysicalAttribution: false, // Preserve the real response-ownership fence at the executor, while avoiding HTTP // and endpoint admission that could conceal an observe-only accounting defect. oauthDispatch: () => async (input, init) => diff --git a/tests/helpers/responses-core-source.ts b/tests/helpers/responses-core-source.ts index b4a250a95fa..508fd6c95e0 100644 --- a/tests/helpers/responses-core-source.ts +++ b/tests/helpers/responses-core-source.ts @@ -58,6 +58,7 @@ export const RESPONSES_CORE_MODULES = [ "response-effects.ts", "response-tool-ownership.ts", "request-send-budget.ts", + "request-spend-target.ts", "account-auto-pause-recovery.ts", "passthrough-execution.ts", "passthrough-dispatch.ts", diff --git a/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts b/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts index bcbc649eebf..95070a6805e 100644 --- a/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts +++ b/tests/providers/github-copilot/github-copilot-auto-runtime.test.ts @@ -5,6 +5,8 @@ import { join } from "node:path"; import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; import type { RequestLogContext } from "../../../src/server/request-log"; import { handleResponses } from "../../../src/server/responses"; +import { handleChatCompletions } from "../../../src/server/chat-completions"; +import { handleClaudeMessages } from "../../../src/server/claude-messages"; import { saveConfig } from "../../../src/config"; import { clearGenericFailoverHealth } from "../../../src/oauth/generic-account-failover"; import { clearCopilotAutoModelsForTests } from "../../../src/providers/github-copilot-auto"; @@ -13,11 +15,18 @@ import { filterCatalogVisibleModels } from "../../../src/codex/catalog/model-vis import { reconcileSuccessfulModelDiscoveries } from "../../../src/providers/new-model-policy"; import type { OcxConfig } from "../../../src/types"; import { createRequestExecutionBudget } from "../../../src/lib/request-execution-budget"; +import { createSpendReservationLedger } from "../../../src/lib/spend-reservation-ledger"; +import { createRequestSpendTracker } from "../../../src/server/responses/request-spend"; +import { createInferenceSendBudget } from "../../../src/server/inference/context"; +import { spendTestJournal, spendTestPolicy, testSpendSalt } from "../../helpers/shipped-spend-ledger"; import { isNonReplayableResponse } from "../../../src/lib/upstream-retry"; import { clearComboSelectionState } from "../../../src/combos/resolve"; -import { clearComboTargetCooldowns } from "../../../src/combos/failover"; +import { clearComboTargetCooldowns, isComboTargetInCooldown } from "../../../src/combos/failover"; +import { clearFallbackEventsForTests, recentFallbackEvents } from "../../../src/routing/fallback-policy"; +import { closeRequestHistoryIndex } from "../../../src/routing/history/indexer"; +import { clearResponseStateForTests, flushResponseState } from "../../../src/responses/state"; import { acquireOwnedSpendHome } from "../../helpers/owned-spend-home"; -import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import { removeTreeAfterBackgroundWork } from "../../helpers/remove-tree"; const previousFetch = globalThis.fetch; const previousHome = process.env.OPENCODEX_HOME; @@ -30,23 +39,34 @@ beforeEach(() => { clearGenericFailoverHealth(); clearComboSelectionState(); clearComboTargetCooldowns(); + clearResponseStateForTests(); release = acquireOwnedSpendHome(); }); -afterEach(() => { - release?.(); - globalThis.fetch = previousFetch; - clearGenericFailoverHealth(); - clearComboSelectionState(); - clearComboTargetCooldowns(); - if (previousHome === undefined) delete process.env.OPENCODEX_HOME; - else process.env.OPENCODEX_HOME = previousHome; - removeTreeWithRetry(home); -}); +afterEach(async () => { + try { + await flushResponseState(); + closeRequestHistoryIndex(); + release?.(); + release = undefined; + await removeTreeAfterBackgroundWork(home); + } finally { + release?.(); + release = undefined; + globalThis.fetch = previousFetch; + closeRequestHistoryIndex(); + clearResponseStateForTests(); + clearGenericFailoverHealth(); + clearComboSelectionState(); + clearComboTargetCooldowns(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + } +}, 35_000); // Background settlement and removal each have a 15s bound, plus scheduling headroom. /** Build an Auto Responses request with caller-supplied history, stream mode, and a lookup tool for wire/continuation assertions. */ -function request(input: unknown, stream = true): Request { +function request(input: unknown, stream = true, overrides: { model?: string; tool_choice?: unknown; tools?: unknown[] } = {}): Request { return new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "github-copilot/auto", input, stream, - tools: [{ type: "function", name: "lookup", parameters: { type: "object", properties: { name: { type: "string" } } } }] }) }); + tools: [{ type: "function", name: "lookup", parameters: { type: "object", properties: { name: { type: "string" } } } }], ...overrides }) }); } function scopedAdmission(config: OcxConfig, allowedModels: string[]) { @@ -66,7 +86,7 @@ async function accounts() { return { a, b }; } /** Return an Auto provider config and send ledger; options drive refusal, expiry, rotation, and tool events through both inference wires. */ -function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; native429?: boolean; onNativeCancel?: () => void; chatRefresh?: boolean; closedNativeBody?: boolean; renewedChatRefusal?: boolean; rotate?: boolean; rotateAt?: number; tool?: boolean; key?: boolean; refusal?: number; negotiationRefusal?: number; negotiationRefusalAt?: number; alwaysRefuse?: boolean; advanceAfterIntent?: () => void | Promise; onSession?: (count: number) => void | Promise; malformedSessionAt?: number; negotiationRetryAfter?: string; replyText?: string; onInference?: (count: number) => void | Promise } = {}) { +function fixture(options: { pauseRefusal?: boolean; chatOnly?: boolean; nativeExpiry?: boolean; nativeFirst?: boolean; native429?: boolean; onNativeCancel?: () => void; chatRefresh?: boolean; closedNativeBody?: boolean; renewedChatRefusal?: boolean; rotate?: boolean; rotateAt?: number; tool?: boolean; key?: boolean; refusal?: number; negotiationRefusal?: number; negotiationRefusalAt?: number; alwaysRefuse?: boolean; advanceAfterIntent?: () => void | Promise; onSession?: (count: number) => void | Promise; malformedSessionAt?: number; negotiationRetryAfter?: string; replyText?: string; onInference?: (count: number) => void | Promise } = {}) { const sent: Array<{ host: string; path: string; token: string | null; body: any; signal?: AbortSignal | null }> = []; let inferenceCount = 0; let sessionCount = 0; @@ -93,7 +113,7 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ expect(headers.get("authorization")).toStartWith(`Bearer synthetic-access-${account}`); sent.push({ host: destination.host, path: destination.pathname, token: headers.get("copilot-session-token"), body, signal: init?.signal }); if (destination.pathname === "/models") return Response.json({ data: [{ id: model, model_picker_enabled: false, - supported_endpoints: [options.nativeExpiry ? (expiredNative ? "/chat/completions" : "/responses") : options.native429 ? (account === "a" ? "/responses" : "/chat/completions") : options.chatRefresh ? (renewed ? "/responses" : "/chat/completions") : options.nativeFirst ? (headers.get("authorization")!.includes("-renewed") ? "/chat/completions" : "/responses") : account === "a" ? "/chat/completions" : "/responses"] }] }); + supported_endpoints: [options.chatOnly ? "/chat/completions" : options.nativeExpiry ? (expiredNative ? "/chat/completions" : "/responses") : options.native429 ? (account === "a" ? "/responses" : "/chat/completions") : options.chatRefresh ? (renewed ? "/responses" : "/chat/completions") : options.nativeFirst ? (headers.get("authorization")!.includes("-renewed") ? "/chat/completions" : "/responses") : account === "a" ? "/chat/completions" : "/responses"] }] }); if (destination.pathname === "/models/session") { sessionCount++; await options.onSession?.(sessionCount); @@ -113,6 +133,9 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ expect(headers.get("authorization")).toBe(`Bearer synthetic-access-${"a"}-renewed`); inferenceCount++; await options.onInference?.(inferenceCount); + if (options.pauseRefusal && inferenceCount === 1) return Response.json({ error: { + type: "permission_error", message: "OAuth authentication is currently not allowed for this organization", + } }, { status: 403 }); if (options.nativeFirst && options.closedNativeBody && inferenceCount === 1) return Response.json({ error: { message: "native-inference-secret" } }, { status: 401 }); if (options.nativeFirst && inferenceCount === 1) return new Response(new ReadableStream({ @@ -126,7 +149,7 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ usage: { input_tokens: 2, output_tokens: 1 } }); if (options.renewedChatRefusal && destination.pathname === "/chat/completions") return Response.json({ error: { message: "must-not-echo" } }, { status: 401 }); - if ((options.nativeFirst || options.nativeExpiry) && !body.stream) return Response.json({ id: "chat-fixture", model, + if ((options.chatOnly || options.nativeFirst || options.nativeExpiry) && !body.stream) return Response.json({ id: "chat-fixture", model, choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], usage: { prompt_tokens: 2, completion_tokens: 1 } }); const tool = options.tool && inferenceCount === (options.nativeFirst ? 2 : 1); @@ -142,6 +165,343 @@ function fixture(options: { nativeExpiry?: boolean; nativeFirst?: boolean; nativ return { config, sent, get cancelledNativeBodies() { return cancelledNativeBodies; }, get refreshCount() { return refreshCount; } }; } describe("Copilot Auto through the Responses pipeline", () => { + test.each(["Chat", "Messages"])("%s retains the local mandatory-tool refusal after Copilot OAuth refresh", async wire => { + await accounts(); + const f = fixture({ chatRefresh: true, chatOnly: true }); + f.config.providers["github-copilot"]!.autoToolChoiceOnlyModels = ["gpt-5.4"]; + const req = new Request(`http://localhost/v1/${wire === "Chat" ? "chat/completions" : "messages"}`, { method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "github-copilot/auto", messages: [{ role: "user", content: "Use lookup." }], stream: false, + ...(wire === "Chat" ? { + tools: [{ type: "function", function: { name: "lookup", parameters: { type: "object", properties: {} } } }], tool_choice: "required", + } : { max_tokens: 64, tools: [{ name: "lookup", input_schema: { type: "object", properties: {} } }], tool_choice: { type: "tool", name: "lookup" } }) }) }); + const response = await (wire === "Chat" ? handleChatCompletions : handleClaudeMessages)(req, f.config, {}); + const body = await response.json(); + expect(response.status).toBe(400); + expect(body.error.message).toContain("mandatory tool_choice cannot be preserved"); + expect(f.sent.filter(call => call.path === "/models/session/intent").map(call => call.host)) + .toEqual(["a.githubcopilot.com", "a.githubcopilot.com"]); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(1); + expect(f.refreshCount).toBe(1); + }); + test("mandatory tool choice uses the negotiated Responses wire instead of the initial Chat route", async () => { + const { b } = await accounts(); + await setActiveAccount("github-copilot", b); + const f = fixture(), choice = { type: "computer_use" }; + const response = await handleResponses(request("hello", false, { tool_choice: choice, tools: [choice] }), f.config, {}); + expect(response.status, await response.text()).toBe(200); + expect(f.sent.some(call => call.path === "/models/session/intent")).toBe(true); + expect(f.sent.filter(call => call.path === "/chat/completions")).toHaveLength(0); + expect(f.sent.find(call => call.path === "/responses")?.body.tool_choice).toEqual(choice); + }); + test("mandatory tool choice refuses a negotiated Chat wire after an initial Responses route", async () => { + await accounts(); + const f = fixture(), choice = { type: "computer_use" }; + const log: RequestLogContext = {}; + const response = await handleResponses(request("hello", true, { + model: "github-copilot/gpt-5.4", tool_choice: choice, tools: [choice], + }), f.config, log); + const body = await response.text(); + expect(response.status).toBe(400); + expect(body).toContain("mandatory tool_choice cannot be preserved"); + expect(f.sent.some(call => call.path === "/models/session/intent")).toBe(true); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + expect(log.model).toBe("gpt-5.4"); + expect(log.activeAttempt?.model).not.toBe("gpt-4o"); + }); + test.each([["combo", false, false], ["policy", false, false], ["combo", true, false], ["policy", true, false], + ["combo", true, true], ["policy", true, true]] as const)( + "mandatory tool choice negotiates a %s child and hops its local refusal (mid-dispatch=%s translated=%s)", async (mode, midDispatch, translated) => { + const { a, b } = await accounts(); + if (midDispatch) await setActiveAccount("github-copilot", translated ? a : b); + const f = fixture(midDispatch ? { chatOnly: translated, + advanceAfterIntent: async () => { await setActiveAccount("github-copilot", translated ? b : a); } } : {}); + if (translated) f.config.providers["github-copilot"]!.autoToolChoiceOnlyModels = ["gpt-5.4"]; + const choice = translated ? { type: "function", name: "lookup" } : { type: "mcp", server_label: "remote", name: "shell" }; + const mcp = { type: "mcp", server_label: "remote", server_url: "https://mcp.example.test", require_approval: "never" }; + const tools = translated ? [{ type: "function", name: "lookup", parameters: { type: "object", properties: {} } }] : [mcp]; + const fallbackBodies: any[] = []; + const output = translated ? [{ id: "call_fixture", type: "function_call", status: "completed", call_id: "lookup_fixture", name: "lookup", arguments: "{}" }] + : [{ id: "mcp_fixture", type: "mcp_call", status: "completed", server_label: "remote", name: "shell", arguments: "{}", output: "done" }]; + f.config.providers.fallback = { adapter: "openai-responses", authMode: "key", apiKey: "fixture", + baseUrl: "https://fallback.example.test/v1", fetch: async (_url, init) => { + fallbackBodies.push(JSON.parse(String(init?.body))); + return Response.json({ id: "resp_fallback", object: "response", status: "completed", model: "fallback-model", output }); + } }; + f.config.combos = { mandatoryAuto: { strategy: "failover", targets: [ + { provider: "github-copilot", model: "auto" }, { provider: "fallback", model: "fallback-model" }, + ] } }; + f.config.routingProfiles = { mandatoryAuto: { candidates: [ + { provider: "github-copilot", model: "auto" }, { provider: "fallback", model: "fallback-model" }, + ] } }; + const log: RequestLogContext = {}; + const spendTargets: Array<{ poolId?: string; identityId?: string }> = []; + const budget = mode === "policy" ? createRequestExecutionBudget(undefined, undefined, { + startRequest: target => { spendTargets.push({ ...target }); }, charge: () => true, refund: () => {}, + }) : undefined; + const response = await handleResponses(request("hello", false, { + model: `${mode}/mandatoryAuto`, tool_choice: choice, tools, + }), f.config, log, { sendBudget: budget }); + const body = await response.json(); + expect(response.status).toBe(200); expect(body.output).toEqual(output); + expect(f.sent.some(call => call.path === "/models/session/intent")).toBe(true); + if (midDispatch) expect(f.sent.filter(call => call.path === "/models/session/intent").map(call => call.host)) + .toEqual(translated ? ["a.githubcopilot.com", "b.githubcopilot.com"] : ["b.githubcopilot.com", "a.githubcopilot.com"]); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + expect(fallbackBodies).toHaveLength(1); expect(fallbackBodies[0].tool_choice).toEqual(choice); + expect(fallbackBodies[0].tools).toEqual(tools); expect(log.provider).toBe(mode === "combo" ? "combo" : "fallback"); + // The generic key route keeps its existing pre-key-commit identity, never Copilot's candidate. + if (mode === "policy") expect(spendTargets.at(-1)).toEqual({ poolId: "fallback", identityId: undefined }); + if (mode === "combo") { + expect(isComboTargetInCooldown("mandatoryAuto", { provider: "github-copilot", model: "auto" })).toBe(false); + clearComboSelectionState(); // Reset selection history, retaining any real target cooldown. + const compatible = await handleResponses(request("hello", true, { model: "combo/mandatoryAuto" }), f.config, {}); + expect(compatible.status).toBe(200); + await compatible.text(); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(1); + expect(fallbackBodies).toHaveLength(1); + } + }); + test("policy fallback treats a post-send renegotiation refusal as a taken hop", async () => { + // The Copilot child sends once, takes a 401, refreshes its OAuth credential, and only THEN + // learns the renewed negotiation settled on a Chat wire that cannot preserve tool_choice. + // That refusal is not a pre-send skip: the hop it rode must commit, and the next candidate's + // hop is guarded FROM github-copilot — the provider scope an operator can switch off. + await accounts(); + const f = fixture({ chatRefresh: true, chatOnly: true }); + f.config.providers["github-copilot"]!.autoToolChoiceOnlyModels = ["gpt-5.4"]; + let primarySends = 0, tailSends = 0; + f.config.providers.primary = { adapter: "openai-chat", authMode: "key", apiKey: "fixture", + baseUrl: "https://primary.example.test/v1", models: ["m1"], fetch: async () => { primarySends++; + return Response.json({ error: { type: "rate_limit_error", message: "limited" } }, { status: 429 }); } }; + f.config.providers.tail = { adapter: "openai-chat", authMode: "key", apiKey: "fixture", + baseUrl: "https://tail.example.test/v1", models: ["m3"], fetch: async () => { tailSends++; + return Response.json({ id: "chat-tail", object: "chat.completion", choices: [{ index: 0, + message: { role: "assistant", content: "tail" }, finish_reason: "stop" }] }); } }; + f.config.fallback = { providers: { "github-copilot": { provider: false } } }; + f.config.routingProfiles = { postSend: { candidates: [ + { provider: "primary", model: "m1" }, + { provider: "github-copilot", model: "auto" }, + { provider: "tail", model: "m3" }, + ] } }; + clearFallbackEventsForTests(); + const response = await handleResponses(request("hello", false, { + model: "policy/postSend", tool_choice: "required", + }), f.config, {}); + const body = await response.json(); + expect(response.status).toBe(409); + expect(body.error.code).toBe("fallback_refused"); + expect(body.error.fallback).toMatchObject({ kind: "provider", scope: "provider", + scopeKey: "github-copilot", from: "github-copilot/auto", to: "tail/m3" }); + expect(f.refreshCount).toBe(1); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(1); + expect(primarySends).toBeGreaterThanOrEqual(1); + expect(tailSends).toBe(0); + expect(recentFallbackEvents().map(event => ({ from: event.from, to: event.to, outcome: event.outcome }))) + .toEqual([ + { from: "primary/m1", to: "github-copilot/auto", outcome: "taken" }, + { from: "github-copilot/auto", to: "tail/m3", outcome: "refused" }, + ]); + }); + test("negotiated Chat refuses required with an empty translated catalog before inference", async () => { + await accounts(); + const f = fixture(); + const response = await handleResponses(request("hello", false, { + tool_choice: "required", tools: [{ type: "computer_use" }], + }), f.config, {}); + expect(response.status, await response.text()).toBe(400); + expect(f.sent.filter(call => call.path === "/models/session/intent")).toHaveLength(1); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + }); + test("a denied combo send reservation retains the local refusal instead of claiming target exhaustion", async () => { + await accounts(); + let reservations = 0; + const budget = createRequestExecutionBudget(undefined, undefined, { + charge: () => ++reservations === 1, refund: () => {}, + }); + const f = fixture(); + let fallbackCalls = 0; + f.config.providers.fallback = { adapter: "openai-responses", authMode: "key", apiKey: "fixture", + baseUrl: "https://fallback.example.test/v1", fetch: async () => { fallbackCalls++; return Response.json({ status: "completed", output: [] }); } }; + f.config.combos = { mandatoryAuto: { strategy: "failover", targets: [ + { provider: "github-copilot", model: "auto" }, { provider: "fallback", model: "fallback-model" }, + ] } }; + const response = await handleResponses(request("hello", false, { + model: "combo/mandatoryAuto", tool_choice: { type: "mcp", server_label: "remote", name: "shell" }, + tools: [{ type: "mcp", server_label: "remote", server_url: "https://mcp.example.test" }], + }), f.config, {}, { sendBudget: budget }); + expect(response.status).toBe(409); + expect(await response.text()).not.toContain("cannot preserve mandatory tool_choice"); + expect(fallbackCalls).toBe(0); + expect(reservations).toBe(2); + expect(budget.used).toBe(0); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + }); + test.each(["combo", "policy"] as const)("%s exhaustion identifies mandatory tool incompatibility without inference", async mode => { + await accounts(); + const f = fixture(); + const targets = [{ provider: "github-copilot", model: "auto" }]; + f.config.combos = { mandatoryAuto: { strategy: "failover", targets } }; + f.config.routingProfiles = { mandatoryAuto: { candidates: targets } }; + const response = await handleResponses(request("hello", false, { + model: `${mode}/mandatoryAuto`, tool_choice: { type: "mcp", server_label: "remote", name: "shell" }, + tools: [{ type: "mcp", server_label: "remote", server_url: "https://mcp.example.test" }], + }), f.config, {}); + const body = await response.json(); + expect(response.status).toBe(mode === "combo" ? 503 : 400); + expect(body.error.message).toContain("mandatory tool_choice"); + expect(f.sent.filter(call => call.path === "/models/session/intent")).toHaveLength(1); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + expect(isComboTargetInCooldown("mandatoryAuto", targets[0]!)).toBe(false); + }); + test.each([["combo", false], ["combo", true], ["policy", false], ["policy", true]] as const)( + "%s mixed exhaustion does not claim all targets refused tools (upstream-first=%s)", async (mode, upstreamFirst) => { + await accounts(); + const f = fixture(); + let fallbackCalls = 0; + f.config.providers.fallback = { adapter: "openai-responses", authMode: "key", apiKey: "fixture", + baseUrl: "https://fallback.example.test/v1", transientRetryOn5xx: { attempts: 1 }, fetch: async () => { + fallbackCalls++; + return Response.json({ error: { type: "upstream_error", message: "synthetic failure" } }, { status: 500 }); + } }; + const targets = [{ provider: "github-copilot", model: "auto" }, { provider: "fallback", model: "fallback-model" }]; + if (upstreamFirst) targets.reverse(); + f.config.combos = { mandatoryAuto: { strategy: "failover", targets } }; + f.config.routingProfiles = { mandatoryAuto: { candidates: targets } }; + const response = await handleResponses(request("hello", false, { + model: `${mode}/mandatoryAuto`, tool_choice: { type: "mcp", server_label: "remote", name: "shell" }, + tools: [{ type: "mcp", server_label: "remote", server_url: "https://mcp.example.test" }], + }), f.config, {}); + expect(response.status).toBe(mode === "combo" && upstreamFirst ? 409 : 500); + expect(await response.text()).not.toContain("cannot preserve mandatory tool_choice"); + expect(fallbackCalls).toBe(1); + expect(f.sent.filter(call => call.path === "/models/session/intent")).toHaveLength(1); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(0); + }); + test.each([false, true])("auto-pause negotiation refusal stays local after native=%s inference", async native => { + const { a, b } = await accounts(); + const log: RequestLogContext = {}; + let sentIdentity: Pick | undefined; + let sentAttemptModel: string | undefined; + const f = fixture({ pauseRefusal: true, native429: native, chatOnly: !native, + onInference: async () => { + sentIdentity = { model: log.model, accountLogLabel: log.accountLogLabel, providerAdapter: log.providerAdapter }; + sentAttemptModel = log.activeAttempt?.model; + await setActiveAccount("github-copilot", b); + } }); + f.config.providers["github-copilot"]!.accountAutoPause = { enabled: true, consecutiveFailures: 1 }; + if (!native) f.config.providers["github-copilot"]!.autoToolChoiceOnlyModels = ["gpt-5.4"]; + saveConfig(f.config); + const choice = native ? { type: "mcp", server_label: "remote", name: "shell" } : { type: "function", name: "lookup" }; + const tools = native ? [{ type: "mcp", server_label: "remote", server_url: "https://mcp.example.test" }] + : [{ type: "function", name: "lookup", parameters: { type: "object", properties: {} } }]; + const budget = createRequestExecutionBudget(); + const response = await handleResponses(request("hello", false, { tool_choice: choice, tools }), f.config, log, { sendBudget: budget }); + expect(response.status).toBe(400); + expect(await response.text()).toContain("mandatory tool_choice cannot be preserved"); + expect(f.sent.filter(call => ["/chat/completions", "/responses"].includes(call.path))).toHaveLength(1); + expect(f.sent.filter(call => call.path === "/models/session/intent").map(call => call.host)) + .toEqual(["a.githubcopilot.com", "b.githubcopilot.com"]); + expect(getAccountSet("github-copilot")!.accounts.find(account => account.id === a)?.paused).toBe(true); + expect(budget.used).toBe(1); + expect(sentIdentity?.model).toBe("gpt-4o"); + expect(log.model).toBe(sentIdentity?.model); + expect(log.accountLogLabel).toBe(sentIdentity?.accountLogLabel); + expect(log.providerAdapter).toBe(sentIdentity?.providerAdapter); + expect(log.activeAttempt?.model).toBe(sentAttemptModel); + expect(getAccountSet("github-copilot")!.activeAccountId).toBe(b); + }); + test.each([false, true])("Copilot funds only ingress-owned roster budgets before sending (ingress=%s)", async ingress => { + await accounts(); + const req = request("hello", false), log: RequestLogContext = {}; + const budget = ingress ? createInferenceSendBudget(req, log) : createRequestExecutionBudget({ + maxTotalModelSends: 2, baseSendAllowance: 2, finalRecoveryAllowance: 0, maxAlternateTargetSends: 1, maxTargetTransitions: 1, + }); + let ceilingAtSend: number | undefined; + const f = fixture({ chatOnly: true, onInference: () => { ceilingAtSend = budget.policy.maxTotalModelSends; } }); + try { + const response = await handleResponses(req, f.config, log, { sendBudget: budget }); + expect(response.status, await response.text()).toBe(200); + expect(f.sent.filter(call => ["/responses", "/chat/completions"].includes(call.path))).toHaveLength(1); + expect(ceilingAtSend).toBe(ingress ? 6 : 2); + expect(budget.policy.maxTotalModelSends).toBe(ingress ? 6 : 2); + } finally { log.spendTracker?.settle(undefined); } + }); + test.each([false, true])("accepted account negotiation preserves physical attribution (deny B=%s)", async deny => { + const { a, b } = await accounts(); + const log: RequestLogContext = { provider: "github-copilot", spendInputEstimateTokens: 1 }; + let firstLabel: string | undefined; + const targets: Array = []; + const tracker = createRequestSpendTracker(log, undefined, createSpendReservationLedger({ + journal: spendTestJournal(), salt: testSpendSalt, policy: spendTestPolicy({ pool: { maxTokens: 100_000 } }), + })); + const budget = createRequestExecutionBudget(undefined, undefined, { ...tracker, + ensureSeed: (target, proof) => { + targets.push(target.identityId); + if (deny && firstLabel && target.identityId && target.identityId !== firstLabel) return undefined; + return tracker.ensureSeed(target, proof); + }, + }); + const f = fixture({ chatOnly: true, pauseRefusal: true, onInference: async count => { + if (count === 1) { firstLabel = log.accountLogLabel; await setActiveAccount("github-copilot", b); } + } }); + f.config.providers["github-copilot"]!.accountAutoPause = { enabled: true, consecutiveFailures: 1 }; + saveConfig(f.config); + try { + const response = await handleResponses(request("hello", false), f.config, log, { sendBudget: budget }); + const text = await response.text(); + expect(response.status, text).toBe(deny ? 429 : 200); + expect(f.sent.filter(call => ["/responses", "/chat/completions"].includes(call.path)).map(call => call.body.model)) + .toEqual(deny ? ["gpt-4o"] : ["gpt-4o", "gpt-5.4"]); + expect(budget.used).toBe(deny ? 1 : 2); + expect((log.attempts ?? []).filter(attempt => attempt.sendCount > 0).map(attempt => attempt.model)) + .toEqual(deny ? ["gpt-4o"] : ["gpt-4o", "gpt-5.4"]); + expect(log.model).toBe(deny ? "gpt-4o" : "gpt-5.4"); + if (deny) expect(log.accountLogLabel).toBe(firstLabel); + else expect(log.accountLogLabel).not.toBe(firstLabel); + expect(targets.some(identity => identity && identity !== firstLabel)).toBe(true); + expect(getAccountSet("github-copilot")!.accounts.find(account => account.id === a)?.paused).toBe(true); + expect(getAccountSet("github-copilot")!.activeAccountId).toBe(b); + } finally { tracker.settle(undefined); } + }); + test.each([false, true])("physical spend denial leaves no Copilot attempt (Chat=%s)", async chat => { + await accounts(); + const f = fixture({ nativeFirst: !chat, chatOnly: chat }); + const budget = createRequestExecutionBudget(undefined, undefined, { + enforced: true, startRequest: () => {}, charge: () => true, refund: () => {}, ensureSeed: () => undefined, + }); + const log: RequestLogContext = {}; + const response = await handleResponses(request("hello", false), f.config, log, { sendBudget: budget }); + expect(response.status).toBe(429); + expect(await response.text()).toContain("request_send_budget_exhausted"); + expect(f.sent.filter(call => ["/responses", "/chat/completions"].includes(call.path))).toHaveLength(0); + expect({ charged: budget.used, physical: budget.physicalStarted, + attempts: (log.attempts ?? []).reduce((total, attempt) => total + attempt.sendCount, 0) }) + .toEqual({ charged: 0, physical: 0, attempts: 0 }); + }); + test.each([[false, false], [true, false], [false, true], [true, true]] as const)( + "pre-physical tool refusal consumes no send or attempt (Chat=%s negotiation401=%s)", async (chat, negotiation401) => { + const { a } = await accounts(); + const actualNow = Date.now; + let offset = 0; + Date.now = () => actualNow() + offset; + try { + const f = fixture({ nativeExpiry: true, chatOnly: chat, advanceAfterIntent: () => { offset = 61_000; }, + ...(negotiation401 ? { negotiationRefusal: 401, negotiationRefusalAt: 2 } : {}) }); + f.config.providers["github-copilot"]!.autoToolChoiceOnlyModels = ["gpt-5.4"]; + const budget = createRequestExecutionBudget(), log: RequestLogContext = {}; + const response = await handleResponses(request("hello", false, { tool_choice: { type: "function", name: "lookup" } }), + f.config, log, { sendBudget: budget }); + expect(response.status).toBe(400); + expect(await response.text()).toContain("mandatory tool_choice cannot be preserved"); + expect(f.sent.filter(call => ["/responses", "/chat/completions"].includes(call.path))).toHaveLength(0); + expect({ charged: budget.used, attempts: (log.attempts ?? []).reduce((total, attempt) => total + attempt.sendCount, 0) }) + .toEqual({ charged: 0, attempts: 0 }); + expect(log.model).toBe("auto"); // No physical send published a negotiated model. + expect(f.refreshCount).toBe(negotiation401 ? 1 : 0); + expect(getAccountSet("github-copilot")!.accounts.find(account => account.id === a)!.credential.access.endsWith("-renewed")) + .toBe(negotiation401); + } finally { Date.now = actualNow; } + }); test.each([[false, false], [true, false], [false, true], [true, true]])( "expired native session hands off to Chat before inference with stream=%s negotiation401=%s", async (stream, negotiation401) => { await accounts(); @@ -190,11 +550,26 @@ describe("Copilot Auto through the Responses pipeline", () => { expect(logCtx.providerAdapter).toBe("openai-chat"); expect(f.sent.filter(call => ["/responses", "/chat/completions"].includes(call.path)).map(call => call.path)) .toEqual(["/responses", "/chat/completions"]); + expect((logCtx.attempts ?? []).filter(attempt => attempt.sendCount > 0) + .map(attempt => [attempt.model, attempt.adapter, attempt.sendCount])) + .toEqual([["gpt-4o", "openai-responses", 1], ["gpt-4o", "openai-chat", 1]]); expect(f.sent.filter(call => call.path === "/models/session")).toHaveLength(2); expect(f.refreshCount).toBe(1); expect(f.cancelledNativeBodies).toBe(1); expect(f.sent.find(call => call.path === "/responses")!.signal!.aborted).toBe(true); }); + test.each([false, true])("same-account model refresh preserves each physical model (stream=%s)", async stream => { + await accounts(); + const f = fixture({ chatRefresh: true, chatOnly: true }), log: RequestLogContext = {}; + const response = await handleResponses(request("hello", stream), f.config, log); + expect(response.status).toBe(200); + expect(await response.text()).toContain("ok"); + expect(f.sent.filter(call => call.path === "/chat/completions").map(call => [call.host, call.body.model])) + .toEqual([["a.githubcopilot.com", "gpt-4o"], ["a.githubcopilot.com", "gpt-5.4"]]); + expect((log.attempts ?? []).filter(attempt => attempt.sendCount > 0).map(attempt => [attempt.model, attempt.adapter, attempt.sendCount])) + .toEqual([["gpt-4o", "openai-chat", 1], ["gpt-5.4", "openai-chat", 1]]); + expect(f.refreshCount).toBe(1); + }); test.each([401, 403, 429])("passthrough refresh negotiation refusal %s stays secret-safe", async status => { await accounts(); const f = fixture({ nativeFirst: true, negotiationRefusal: status, negotiationRefusalAt: 2 }); diff --git a/tests/responses/responses-account-auto-pause.test.ts b/tests/responses/responses-account-auto-pause.test.ts index ff99c31ab95..8a4e502e668 100644 --- a/tests/responses/responses-account-auto-pause.test.ts +++ b/tests/responses/responses-account-auto-pause.test.ts @@ -1,18 +1,63 @@ -import { afterEach, beforeEach, expect, test } from "bun:test"; +import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { getDefaultConfig, saveConfig } from "../../src/config"; import { getAccountSet, saveCredential, setActiveAccount } from "../../src/oauth/store"; import { handleResponses } from "../../src/server/responses/core"; +import * as transport from "../../src/server/responses/request-transport"; import { flushResponseState, clearResponseStateForTests } from "../../src/responses/state"; import { clearGenericFailoverHealth } from "../../src/oauth/generic-account-failover"; import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import type { OcxConfig } from "../../src/types"; +import { createAutoPauseRefreshErrorMapper, recoverAutoPausedAccount } from "../../src/server/responses/account-auto-pause-recovery"; +import { createResponsesSendBudget } from "../../src/server/responses/request-send-budget"; +import { createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { isPolicyCandidateRefusal, PolicyCandidateUnavailableError, policyCandidateRefusalResponse } from "../../src/server/responses/policy-request-scope"; const originalHome = process.env.OPENCODEX_HOME; const originalFetch = globalThis.fetch; +for (const outcome of ["known", "unknown", "cancelled"] as const) test(`auto-pause ${outcome} refresh failure disposes its exchange before projection`, async () => { + const upstream = new AbortController(); + const caller = new AbortController(); + let cancels = 0, unlinked = 0, mapped = 0; + const original = new Response(new ReadableStream({ cancel() { cancels++; } })); + const sentinel = new Error("unknown refresh failure"); + const refusal = policyCandidateRefusalResponse(new PolicyCandidateUnavailableError())!; + if (outcome === "cancelled") caller.abort(); + const mapper = createAutoPauseRefreshErrorMapper(original, upstream, () => { unlinked++; }, + error => { mapped++; return error === sentinel ? undefined : refusal; }, + () => caller.signal.aborted, () => new Response(null, { status: 499 })); + const result = mapper(outcome === "unknown" ? sentinel : new PolicyCandidateUnavailableError()); + await Promise.resolve(); + expect(cancels).toBe(1); expect(unlinked).toBe(1); expect(upstream.signal.aborted).toBe(true); + if (outcome === "unknown") expect(result).toBeUndefined(); + else expect(result?.failed.status).toBe(outcome === "cancelled" ? 499 : 409); + expect(mapped).toBe(outcome === "cancelled" ? 0 : 1); + if (outcome === "known") expect(isPolicyCandidateRefusal(result!.failed)).toBe(true); + await refusal.body?.cancel(); +}); +for (const mapped of [true, false]) test(`auto-pause refresh refusal ${mapped ? "maps to its local response" : "preserves unknown errors"} and refunds the unused hop`, async () => { + const budget = createRequestExecutionBudget(); + const state = createResponsesSendBudget({ options: { sendBudget: budget }, logCtx: { provider: "fixture" }, + req: new Request("http://fixture/v1/responses") }); + if (state instanceof Response) throw new Error("fixture budget refused"); + const error = mapped ? new PolicyCandidateUnavailableError() : new Error("unmapped refresh failure"); + let refetches = 0; + const result = recoverAutoPausedAccount({ requestKey: {}, reserve: state.reserveCredentialHop, + refresh: async () => { expect(budget.used).toBe(1); throw error; }, + mapRefreshError: policyCandidateRefusalResponse, + refetch: async () => { refetches++; return Response.json({ unexpected: true }); }, + setPermit: permit => { state.pendingHopPermit = permit; }, + }); + if (mapped) { + const response = await result; + expect(response?.status).toBe(409); expect(isPolicyCandidateRefusal(response!)).toBe(true); + await response?.body?.cancel(); + } else await expect(result).rejects.toBe(error); + expect(refetches).toBe(0); expect(budget.used).toBe(0); expect(state.pendingHopPermit).toBeUndefined(); +}); let home: string; let releaseSpend: () => void; beforeEach(() => { @@ -39,11 +84,52 @@ function configFor(name: string, oauth: boolean): OcxConfig { saveConfig(config); return config; } -async function send(config: OcxConfig, name: string, sendBudget?: import("../../src/lib/request-execution-budget").RequestExecutionBudget) { - const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: `${name}/fixture`, input: "synthetic", stream: false, store: false }) }), config, { model: "fixture", provider: name }, { sendBudget }); +async function send(config: OcxConfig, name: string, sendBudget?: import("../../src/lib/request-execution-budget").RequestExecutionBudget, abortSignal?: AbortSignal, requestSignal?: AbortSignal) { + const response = await handleResponses(new Request("http://localhost/v1/responses", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ model: `${name}/fixture`, input: "synthetic", stream: false, store: false }), signal: requestSignal }), config, { model: "fixture", provider: name }, { sendBudget, abortSignal }); const text = await response.text(); return { status: response.status, text }; } +for (const native of [false, true]) for (const outcome of ["known", "unknown", "cancelled", "request-aborted"] as const) +test(`auto-pause ${native ? "native" : "translated"} recovery settles a ${outcome} refresh failure without another send`, async () => { + const config = configFor("fixture", false); + config.providers.fixture!.adapter = native ? "openai-responses" : "openai-chat"; + config.providers.fixture!.accountAutoPause = { enabled: true, consecutiveFailures: 1 }; + saveConfig(config); + const budget = createRequestExecutionBudget(), sends: string[] = [], signals: AbortSignal[] = []; + const caller = new AbortController(), requestController = new AbortController(), sentinel = new Error("original refresh error"); + globalThis.fetch = (async (_input, init) => { + sends.push(new Headers(init?.headers).get("authorization")!); + if (init?.signal) signals.push(init.signal); + return orgRefusal(); + }) as typeof fetch; + let refreshes = 0; + const prepare = transport.prepareResponsesTransport; + const prepared = spyOn(transport, "prepareResponsesTransport").mockImplementation(async (...args) => { + const state = await prepare(...args); + if (!(state instanceof Response)) state.recoverAutoPausedSelection = async () => { + refreshes++; + if (outcome === "cancelled") caller.abort(); + if (outcome === "request-aborted") requestController.abort(); + throw outcome === "unknown" ? sentinel : new PolicyCandidateUnavailableError(); + }; + return state; + }); + try { + const result = send(config, "fixture", budget, caller.signal, requestController.signal); + if (outcome === "unknown") await expect(result).rejects.toBe(sentinel); + else { + const response = await result; + expect(response.status).toBe(outcome === "cancelled" ? 499 : 400); + if (outcome !== "cancelled") expect(response.text).toContain("mandatory tool_choice cannot be preserved"); + } + expect(refreshes).toBe(1); expect(sends).toEqual(["Bearer synthetic-A"]); + expect(signals).toHaveLength(1); expect(signals[0]!.aborted).toBe(true); + expect(config.providers.fixture!.apiKeyPool![0]!.disabled).toBe(true); + expect(budget.used).toBe(1); + expect(caller.signal.aborted).toBe(outcome === "cancelled"); + expect(requestController.signal.aborted).toBe(outcome === "request-aborted"); + } finally { prepared.mockRestore(); } +}); for (const oauth of [true, false]) { test(`${oauth ? "OAuth" : "key"} HTTP path pauses at threshold and serves the same request with its sibling`, async () => { const name = oauth ? "command-code" : "fixture"; diff --git a/tests/responses/responses-direct-chat-tool-choice.test.ts b/tests/responses/responses-direct-chat-tool-choice.test.ts new file mode 100644 index 00000000000..8a86ec7ed62 --- /dev/null +++ b/tests/responses/responses-direct-chat-tool-choice.test.ts @@ -0,0 +1,203 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { clearResponseStateForTests, flushResponseState } from "../../src/responses/state"; +import { closeRequestHistoryIndex } from "../../src/routing/history/indexer"; +import { handleResponses } from "../../src/server/responses"; +import type { RequestLogContext } from "../../src/server/request-log"; +import type { OcxConfig } from "../../src/types"; +import { acquireOwnedSpendHome } from "../helpers/owned-spend-home"; +import { createTempHome } from "../helpers/temp-home"; + +const clientTool = { type: "function", name: "shell", parameters: { type: "object", properties: {} } }; +const mcpTool = { type: "mcp", server_label: "remote", server_url: "https://mcp.fixture.invalid/mcp" }; +const mcpChoice = { type: "allowed_tools", mode: "required", tools: [{ type: "mcp", server_label: "remote", name: "shell" }] }; +const originalFetch = globalThis.fetch; +let home: ReturnType; +let releaseHome: () => void; + +beforeEach(() => { + home = createTempHome("ocx-direct-chat-choice-"); + releaseHome = acquireOwnedSpendHome(); + clearResponseStateForTests(); + globalThis.fetch = (() => { throw new Error("unexpected network access in tool-choice fixture"); }) as typeof fetch; +}); + +afterEach(async () => { + globalThis.fetch = originalFetch; + await flushResponseState(); + clearResponseStateForTests(); + closeRequestHistoryIndex(); + releaseHome(); + home.remove(); +}); + +function fixture(nativeOutput: Record[] = [{ id: "computer_fixture", type: "computer_call", status: "completed" }]) { + const sent: Array<{ provider: string; body: Record }> = []; + const send = (provider: string) => (async (_input, init) => { + sent.push({ provider, body: JSON.parse(String(init?.body)) }); + return Response.json(provider === "chat" ? { + id: "chat_fixture", object: "chat.completion", model: "model", + choices: [{ index: 0, finish_reason: "tool_calls", message: { role: "assistant", content: null, + tool_calls: [{ id: "call_fixture", type: "function", function: { name: "shell", arguments: "{}" } }], + } }], + usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 }, + } : { + id: "resp_fixture", object: "response", status: "completed", model: "model", + output: nativeOutput, + usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 }, + }); + }) as typeof fetch; + const config: OcxConfig = { + port: 0, defaultProvider: "chat", multiAgentGuidanceEnabled: false, + providers: { + chat: { adapter: "openai-chat", baseUrl: "https://api.openai.com/v1", authMode: "key", + apiKey: "fixture-key", models: ["model"], liveModels: false, fetch: send("chat") }, + responses: { adapter: "openai-responses", baseUrl: "https://responses.fixture.invalid/v1", authMode: "key", + apiKey: "fixture-key", models: ["model"], liveModels: false, fetch: send("responses") }, + }, + routingProfiles: { tools: { candidates: [{ provider: "chat", model: "model" }, { provider: "responses", model: "model" }] } }, + }; + return { config, sent }; +} + +function request(toolChoice: unknown, tools: unknown[] | undefined, model = "chat/model", input: unknown = "Use the selected tool."): Request { + return new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model, input, stream: false, store: false, tools, tool_choice: toolChoice }), + }); +} + +test("direct translated Chat rejects an unrepresentable computer_use choice before send", async () => { + const { config, sent } = fixture(); + const response = await handleResponses(request({ type: "computer_use" }, [{ type: "computer_use" }, clientTool]), config, {}); + expect(response.status).toBe(400); + expect(await response.text()).toContain("mandatory tool_choice cannot be preserved"); + expect(sent).toEqual([]); +}); + +test.each(["computer_use", "computer_use_preview", "file_search", "code_interpreter"])( + "required with only a dropped %s tool is refused before Chat sends", async type => { + const { config, sent } = fixture(); + const response = await handleResponses(request("required", [{ type }]), config, {}); + expect(response.status, await response.text()).toBe(400); + expect(sent).toEqual([]); + }, +); + +test("required still accepts a mixed catalog when a declared client tool survives", async () => { + const { config, sent } = fixture(); + const response = await handleResponses(request("required", [{ type: "computer_use" }, clientTool]), config, {}); + expect(response.status, await response.text()).toBe(200); + expect(sent).toHaveLength(1); + expect(sent[0]!.body.tool_choice).toBe("required"); + expect((sent[0]!.body.tools as Array<{ function: { name: string } }>).map(tool => tool.function.name)).toEqual(["shell"]); +}); + +test("a policy skips an empty translated required catalog and retains its native tools", async () => { + const { config, sent } = fixture(); + const tools = [{ type: "computer_use" }]; + const response = await handleResponses(request("required", tools, "policy/tools"), config, {}); + expect(response.status, await response.text()).toBe(200); + expect(sent.map(row => row.provider)).toEqual(["responses"]); + expect(sent[0]!.body.tool_choice).toBe("required"); + expect(sent[0]!.body.tools).toEqual(tools); +}); + +for (const type of ["additional_tools", "tool_search_output"]) for (const model of ["chat/model", "policy/tools"]) { + test(`required retains a client tool loaded from ${type} on ${model}`, async () => { + const { config, sent } = fixture(); + const input = [{ type, role: "developer", call_id: "search-fixture", status: "completed", tools: [clientTool] }, + { role: "user", content: "Use shell." }]; + const response = await handleResponses(request("required", [{ type: "file_search" }], model, input), config, {}); + expect(response.status, await response.text()).toBe(200); + expect(sent.map(row => row.provider)).toEqual(["chat"]); + expect((sent[0]!.body.tools as Array<{ function: { name: string } }>).map(tool => tool.function.name)).toEqual(["shell"]); + }); +} + +for (const type of ["additional_tools", "tool_search_output"]) for (const model of ["chat/model", "policy/tools"]) + for (const topLevel of [undefined, []]) { + test(`required input-only ${type} rejects empty Chat tools on ${model}, top=${String(topLevel)}`, async () => { + const { config, sent } = fixture([{ id: "file-fixture", type: "file_search_call", status: "completed" }]); + const input = [{ type, role: "developer", call_id: "search-fixture", status: "completed", tools: [{ type: "file_search" }] }, + { role: "user", content: "Use the loaded tool." }]; + const response = await handleResponses(request("required", topLevel, model, input), config, {}); + expect(response.status, await response.text()).toBe(model === "chat/model" ? 400 : 200); + expect(sent.map(row => row.provider)).toEqual(model === "chat/model" ? [] : ["responses"]); + if (sent.length) { + expect(sent[0]!.body.tool_choice).toBe("required"); + const forwarded = sent[0]!.body.input as Array>; + if (type === "tool_search_output") { + // The native adapter lowers this carrier to its supported result envelope. + expect(forwarded[0]!.type).toBe("function_call_output"); + expect(forwarded[0]!.call_id).toBe("search-fixture"); + expect(JSON.parse(String(forwarded[0]!.output))).toEqual({ tools: [{ type: "file_search" }], status: "completed" }); + } else expect(forwarded).toEqual(input); + } + }); + } + +test("native Chat cannot replace a required named MCP tool with a same-named client function", async () => { + const { config, sent } = fixture(); + const response = await handleResponses(request(mcpChoice, [mcpTool, clientTool]), config, {}); + expect(response.status, await response.text()).toBe(400); + expect(sent).toEqual([]); +}); + +test.each(["required", { type: "function", name: "shell" }, { + type: "allowed_tools", mode: "required", tools: [{ type: "function", name: "shell" }], +}])("representable client tool choice %j reaches Chat unchanged", async choice => { + const { config, sent } = fixture(); + const response = await handleResponses(request(choice, [clientTool]), config, {}); + const body = await response.json(); + expect(response.status).toBe(200); + expect(body.status).toBe("completed"); + expect(sent).toHaveLength(1); + expect(sent[0]!.provider).toBe("chat"); + expect(sent[0]!.body.tool_choice).toEqual(choice === "required" ? "required" : { type: "function", function: { name: "shell" } }); +}); + +test("a policy skips incompatible Chat and preserves the mandatory selector for Responses", async () => { + const { config, sent } = fixture(); + const choice = { type: "computer_use" }; + const log: RequestLogContext = {}; + const response = await handleResponses(request(choice, [{ type: "computer_use" }, clientTool], "policy/tools"), config, log); + expect(response.status, await response.text()).toBe(200); + expect(log.routeDecision?.selected.provider).toBe("chat"); + expect(sent.map(row => row.provider)).toEqual(["responses"]); + expect(log.provider).toBe("responses"); + expect(sent[0]!.body.tool_choice).toEqual(choice); +}); + +test("a shadow-intercepted policy route retries a compatible candidate on the real endpoint", async () => { + // The intercept rewrites the model to the profile AFTER the body's parse hook, so the retry + // snapshot must exist before interception — otherwise the captured policy route skips + // traversal and the caller sees the internal candidate refusal instead of this hop. + const { config, sent } = fixture(); + config.shadowCallIntercept = { enabled: true, model: "policy/tools", sourceModels: ["shadow-helper"] }; + const choice = { type: "computer_use" }; + const log: RequestLogContext = {}; + const response = await handleResponses( + request(choice, [{ type: "computer_use" }, clientTool], "shadow-helper-x"), config, log); + expect(response.status, await response.text()).toBe(200); + expect(log.shadowCallRewrittenFrom).toBe("shadow-helper"); + expect(log.routeDecision?.selected.provider).toBe("chat"); + expect(sent.map(row => row.provider)).toEqual(["responses"]); + expect(sent[0]!.body.tool_choice).toEqual(choice); +}); + +test("a policy preserves required MCP ownership across translator-budget fallback", async () => { + const output = [{ id: "mcp_fixture", type: "mcp_call", status: "completed", server_label: "remote", name: "shell", arguments: "{}", output: "done" }]; + const { config, sent } = fixture(output); + const log: RequestLogContext = {}; + const response = await handleResponses(request(mcpChoice, [mcpTool, clientTool], "policy/tools"), config, log); + const body = await response.json(); + expect(response.status).toBe(200); + expect(body.status).toBe("completed"); + expect(log.routeDecision?.selected.provider).toBe("chat"); + expect(sent.map(row => row.provider)).toEqual(["responses"]); + expect(log.provider).toBe("responses"); + expect(sent[0]!.body.tool_choice).toEqual(mcpChoice); + expect(sent[0]!.body.tools).toEqual([mcpTool, clientTool]); + expect(body.output).toEqual(output); +}); diff --git a/tests/responses/responses-fetch-helpers-boundary.test.ts b/tests/responses/responses-fetch-helpers-boundary.test.ts index 713ca37346e..0509e1a41da 100644 --- a/tests/responses/responses-fetch-helpers-boundary.test.ts +++ b/tests/responses/responses-fetch-helpers-boundary.test.ts @@ -5,10 +5,31 @@ import { fileURLToPath } from "node:url"; import { createScanner, LanguageVariant, SyntaxKind } from "typescript/unstable/ast"; import { fetchWithHeaderTimeout, providerFetch, sendWithConnectionPolicy, storedPoolReplayDispatchNotifier } from "../../src/server/responses/fetch-helpers"; import { repoRoot as resolveRepoRoot } from "../helpers/repo-root"; +import { createPhysicalSendReporter, createRequestExecutionBudget } from "../../src/lib/request-execution-budget"; +import { errorDestinationLabel, SendBudgetExhaustedError } from "../../src/lib/upstream-retry"; +import { physicalSendAccounting } from "../../src/server/responses/deferred-send-reporter"; const repoRoot = resolveRepoRoot(); const helperPath = resolve(repoRoot, "src/server/responses/fetch-helpers.ts"); +test("deferred accounting rejects reuse before counting an already consumed reservation", () => { + const budget = createRequestExecutionBudget(); + const reservation = budget.reserveDispatch({ sendClass: "initial", targetKey: "fixture", countedExternally: true }); + if (!reservation.allowed) throw new Error("fixture reservation was refused"); + const accounting = physicalSendAccounting(createPhysicalSendReporter(budget, () => ({ poolId: "fixture" }), reservation.permit), { + permit: reservation.permit, destination: () => errorDestinationLabel("https://fixture.example/v1"), + }); + try { + accounting.onPhysicalDispatch!(); + accounting.report(1); + expect(budget.used).toBe(1); + expect(() => accounting.onPhysicalDispatch!()).toThrow(SendBudgetExhaustedError); + accounting.report(1); + expect(budget.used).toBe(1); + } finally { accounting.report.close?.(); reservation.permit.release(); } + expect(budget.used).toBe(1); +}); + test("an unpaced physical send does not yield after its caller's credential validation", async () => { const order: string[] = []; queueMicrotask(() => order.push("credential-change")); diff --git a/tests/responses/responses-passthrough-transient-policy.test.ts b/tests/responses/responses-passthrough-transient-policy.test.ts index 8c50905c87e..f62c5fb13de 100644 --- a/tests/responses/responses-passthrough-transient-policy.test.ts +++ b/tests/responses/responses-passthrough-transient-policy.test.ts @@ -57,10 +57,12 @@ describe("the Responses passthrough lane reads the provider transient policy", ( const allowanceBacked = occurrences(packed, "attempts:allowance.attempts"); const prepaidInitial = occurrences(packed, "attempts:sendBudgetState.initialSendAllowance(transientSendAttempts())"); expect(prepaidInitial).toBe(1); - // Combo-owned, non-enforced sends settle at physical dispatch; every other send keeps the - // shared reporter, so exactly one of the two accounts for any given send. - expect(packed).toContain("onPhysicalDispatch:receiptMode?sendBudgetState.noteInitialDispatch:undefined"); - expect(packed).toContain("...(receiptMode?{}:{onSendsConsumed:transientSendReporter()})"); + // Combo receipt mode settles directly; other sends keep the shared reporter. Copilot's + // reporter defers its charge until physical dispatch, after negotiation can refuse locally. + expect(packed).toContain("constinitialAccounting=copilotSendAccounting()"); + expect(packed).toContain("onPhysicalDispatch:receiptMode?sendBudgetState.noteInitialDispatch:initialAccounting.onPhysicalDispatch"); + expect(packed).toContain("...(receiptMode?{}:{onSendsConsumed:initialAccounting.report})"); + expect(packed).toContain("=>physicalSendAccounting(transientSendReporter(permit),"); expect(budgeted + allowanceBacked + prepaidInitial).toBe(sites); // The rebuild leg spends a recovery allowance rather than the base budget directly, so its diff --git a/tests/routing/routing-policy-fallback.test.ts b/tests/routing/routing-policy-fallback.test.ts index 27564ab303c..431ae6553ce 100644 --- a/tests/routing/routing-policy-fallback.test.ts +++ b/tests/routing/routing-policy-fallback.test.ts @@ -1,12 +1,16 @@ import { describe, expect, test } from "bun:test"; import { formatErrorResponse } from "../../src/bridge"; +import { isProxyError } from "../../src/lib/client-error-body"; +import { finalizeAccountLease } from "../../src/server/responses/core-lifetime"; +import { MandatoryToolChoiceUnavailableError, PolicyCandidateUnavailableError, policyCandidateRefusalResponse } from "../../src/server/responses/policy-request-scope"; import { RequestPacingQueueOverloadError } from "../../src/providers/request-pacing"; import { fetchWithTransientRetry, isNonReplayableResponse, markResponseNonReplayable } from "../../src/lib/upstream-retry"; import { shouldRetryCodexPoolAccountQuota } from "../../src/server/responses/core-codex-account"; import type { OcxConfig } from "../../src/types"; -import { beginRequestAttempt, type RequestLogContext } from "../../src/server/request-log"; +import { beginRequestAttempt, noteAttemptSend, type RequestLogContext } from "../../src/server/request-log"; import type { RouteDecisionTraceV1 } from "../../src/routing/trace"; +import { clearFallbackEventsForTests, recentFallbackEvents } from "../../src/routing/fallback-policy"; import { fakeChatGptJwt } from "../helpers/fake-chatgpt-jwt"; import { parseSyntheticRowId } from "../../src/server/fast-row"; import { @@ -61,6 +65,139 @@ function seedAttempt(logCtx: RequestLogContext, provider: string, model: string) } describe("policy candidate fallback", () => { + test.each([false, true])("upstream failure body settles when the next policy hop is refused=%s", async denied => { + const config = fixtureConfig(); + if (denied) config.fallback = { routes: { "policy/daily": { provider: false } } }; + let calls = 0, released = 0; + const original = finalizeAccountLease(formatErrorResponse(503, "upstream_error", "synthetic failure"), () => { released++; }); + const success = Response.json({ status: "completed" }); + const response = await handleResponsesWithPolicyFallback(request(), config, {}, {}, { + runCore: async (req, _config, context, options) => { + calls++; + options.onRequestBodyParsed?.(await req.json()); + context.routeDecision = policyTrace(); + return calls === 1 ? original : success; + }, + }); + try { + expect(calls).toBe(denied ? 1 : 2); + expect(response.status).toBe(denied ? 503 : 200); + expect(original.bodyUsed).toBe(true); + expect(released).toBe(1); + expect(response.bodyUsed).toBe(false); + const body = await response.json(); + if (denied) expect(body.error.code).toBe("fallback_refused"); + else expect(body.status).toBe("completed"); + expect(released).toBe(1); + } finally { await original.body?.cancel().catch(() => {}); await response.body?.cancel().catch(() => {}); await success.body?.cancel().catch(() => {}); } + }); + test.each([ + [["tool", "tool", "tool"], 400], + [["tool", "generic", "tool"], 409], + [["generic", "tool", "tool"], 409], + [["tool", "upstream", "tool"], 500], + [["upstream", "tool", "tool"], 500], + ] as const)("exhaustion preserves mixed failures and settles discarded local bodies (%s)", async (kinds, status) => { + let calls = 0; + const released = [0, 0, 0], bodies: Response[] = []; + const response = await handleResponsesWithPolicyFallback(request(), fixtureConfig(), {}, {}, { + runCore: async (req, _config, context, options) => { + const index = calls++, kind = kinds[index]; + options.onRequestBodyParsed?.(await req.json()); + context.routeDecision = policyTrace(); + const source = kind === "upstream" ? formatErrorResponse(500, "upstream_error", "synthetic failure") + : policyCandidateRefusalResponse(kind === "tool" ? new MandatoryToolChoiceUnavailableError() : new PolicyCandidateUnavailableError())!; + const body = finalizeAccountLease(source, () => { released[index]++; }); + bodies.push(body); + return body; + }, + }); + expect(calls).toBe(3); + expect(response.status).toBe(status); + const text = await response.text(); + expect(text.includes("mandatory tool_choice")).toBe(status === 400); + if (status === 400) expect(isProxyError(response)).toBe(true); + expect(released).toEqual([1, 1, 1]); + expect(bodies.every(body => body.bodyUsed)).toBe(true); + }); + test("a refusal minted after a physical send is a taken hop, not a skip", async () => { + // A re-negotiated destination can refuse only after its send — a Copilot credential or + // account recovery settles the wire mid-dispatch. That refusal is not a local skip: the + // hop it rode commits, and the sent destination becomes the next hop's source, which is + // where a provider-scoped fallback refusal keys on. + const trace = policyTrace(); + const config = { + ...fixtureConfig(), + fallback: { providers: { "provider-b": { provider: false } } }, + } as OcxConfig; + const logCtx = { requestedModel: "policy/daily", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; + const seenModels: string[] = []; + clearFallbackEventsForTests(); + const response = await handleResponsesWithPolicyFallback(request(), config, logCtx, {}, { + runCore: async (req, _config, childLog, options) => { + const body = await req.json() as { model: string }; + options.onRequestBodyParsed?.(body); + seenModels.push(body.model); + childLog.routeDecision = trace; + if (seenModels.length === 1) { + seedAttempt(childLog, "provider-a", "model-a"); + return Response.json({ error: { type: "rate_limit_error" } }, { status: 429 }); + } + seedAttempt(childLog, "provider-b", "model-b"); + noteAttemptSend(childLog.activeAttempt, undefined); + return policyCandidateRefusalResponse(new MandatoryToolChoiceUnavailableError())!; + }, + }); + + expect(response.status).toBe(409); + expect(seenModels).toEqual(["policy/daily", "provider-b/model-b"]); + const body = await response.json() as { error: { code: string; fallback?: Record } }; + expect(body.error.code).toBe("fallback_refused"); + expect(body.error.fallback).toMatchObject({ + kind: "provider", scope: "provider", scopeKey: "provider-b", + from: "provider-b/model-b", to: "provider-c/model-c", + }); + expect(recentFallbackEvents().map(event => ({ from: event.from, to: event.to, outcome: event.outcome }))).toEqual([ + { from: "provider-a/model-a", to: "provider-b/model-b", outcome: "taken" }, + { from: "provider-b/model-b", to: "provider-c/model-c", outcome: "refused" }, + ]); + }); + test("a refusal minted before any send still skips without committing its hop", async () => { + // Behavior pin for the seam's other side: a candidate refused before its first physical + // send remains a local skip. Its hop is never committed, the prior failure is retained, + // and the next hop is still evaluated from the candidate that actually failed before it. + const trace = policyTrace(); + const config = { + ...fixtureConfig(), + fallback: { providers: { "provider-b": { provider: false } } }, + } as OcxConfig; + const logCtx = { requestedModel: "policy/daily", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; + const seenModels: string[] = []; + clearFallbackEventsForTests(); + const response = await handleResponsesWithPolicyFallback(request(), config, logCtx, {}, { + runCore: async (req, _config, childLog, options) => { + const body = await req.json() as { model: string }; + options.onRequestBodyParsed?.(body); + seenModels.push(body.model); + childLog.routeDecision = trace; + if (seenModels.length === 1) { + seedAttempt(childLog, "provider-a", "model-a"); + return Response.json({ error: { type: "rate_limit_error" } }, { status: 429 }); + } + if (seenModels.length === 2) { + seedAttempt(childLog, "provider-b", "model-b"); + return policyCandidateRefusalResponse(new MandatoryToolChoiceUnavailableError())!; + } + return Response.json({ status: "completed" }); + }, + }); + + expect(response.status).toBe(200); + expect(seenModels).toEqual(["policy/daily", "provider-b/model-b", "provider-c/model-c"]); + expect(recentFallbackEvents().map(event => ({ from: event.from, to: event.to, outcome: event.outcome }))).toEqual([ + { from: "provider-a/model-a", to: "provider-c/model-c", outcome: "taken" }, + ]); + }); test.each([400, 422])("HTTP %s reflected free-tier prose does not authorize a policy fallback", async status => { let calls = 0; const failure = formatErrorResponse(status, "invalid_request_error", "Invalid value: free tier allows a single request", { @@ -740,4 +877,158 @@ describe("policy candidate fallback", () => { expect(await response.text()).toContain("hello"); expect(calls).toBe(1); }); + + test("a shadow-intercepted policy route retries the next candidate with the original body", async () => { + // The intercept rewrites the model AFTER the body-parsed hook, so the snapshot gate cannot + // key on the destination. Without the snapshot this request's captured policy route loses + // traversal entirely and the caller receives the internal 409 instead of the next candidate. + const trace = policyTrace(); + const config = { + ...fixtureConfig(), + shadowCallIntercept: { enabled: true, model: "policy/daily", sourceModels: ["shadow-helper"] }, + } as OcxConfig; + const logCtx = { requestedModel: "shadow-helper-x", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; + const seenModels: string[] = []; + const seenIsolated: boolean[] = []; + const seenCandidates: Array = []; + let retriedBody: Record | undefined; + const req = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + model: "shadow-helper-x", input: "hello", stream: false, + tool_choice: { type: "function", name: "get_value" }, + tools: [{ type: "function", name: "get_value" }], + }), + }); + const response = await handleResponsesWithPolicyFallback(req, config, logCtx, {}, { + runCore: async (attempt, _config, ctx, options) => { + const body = await attempt.json() as Record & { model: string }; + options.onRequestBodyParsed?.(body); + seenModels.push(body.model); + seenIsolated.push(options.shadowCallIntercepted === true); + seenCandidates.push(options.policyFallbackCandidate === undefined + ? undefined : `${options.policyFallbackCandidate.provider}/${options.policyFallbackCandidate.model}`); + ctx.routeDecision = trace; + if (seenModels.length === 1) { + // What preparation records when interception rewrites to a policy target: the shared + // log carries the configured source prefix, the scope carries the captured decision. + ctx.shadowCallRewrittenFrom = "shadow-helper"; + seedAttempt(ctx, "provider-a", "model-a"); + // Emulate preparation's in-place normalization on the parsed body: the retry must + // replay the immutable snapshot, never these mutations. + body.model = "policy/daily"; + body.input = "mutated by preparation"; + (body.tools as unknown[]).push({ type: "function", name: "injected_tool" }); + return policyCandidateRefusalResponse(new MandatoryToolChoiceUnavailableError())!; + } + retriedBody = body; + seedAttempt(ctx, "provider-b", "model-b"); + return Response.json({ status: "completed" }); + }, + }); + expect(response.status).toBe(200); + expect(seenModels).toEqual(["shadow-helper-x", "provider-b/model-b"]); + expect(seenCandidates).toEqual([undefined, "provider-b/model-b"]); + // The replay is the immutable caller body — original tools and tool_choice, not the + // post-interception mutations the first attempt's preparation made in place. + expect(retriedBody).toMatchObject({ + model: "provider-b/model-b", + input: "hello", + tool_choice: { type: "function", name: "get_value" }, + }); + expect(retriedBody!.tools).toEqual([{ type: "function", name: "get_value" }]); + // The retried candidate keeps helper isolation and the suppressed selector label through + // the existing option; the first-party attempt left it unset. + expect(seenIsolated).toEqual([false, true]); + }); + + test("a shadow-intercepted policy route exhausts candidates to the client-facing 400", async () => { + const trace = policyTrace(); + const config = { + ...fixtureConfig(), + shadowCallIntercept: { enabled: true, model: "policy/daily", sourceModels: ["shadow-helper"] }, + } as OcxConfig; + const logCtx = { requestedModel: "shadow-helper-x", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; + const seenModels: string[] = []; + const req = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "shadow-helper-x", input: "hello", stream: false }), + }); + const response = await handleResponsesWithPolicyFallback(req, config, logCtx, {}, { + runCore: async (attempt, _config, ctx, options) => { + const body = await attempt.json() as { model: string }; + options.onRequestBodyParsed?.(body); + seenModels.push(body.model); + ctx.routeDecision = trace; + ctx.shadowCallRewrittenFrom = "shadow-helper"; + seedAttempt(ctx, "provider", body.model); + return policyCandidateRefusalResponse(new MandatoryToolChoiceUnavailableError())!; + }, + }); + expect(seenModels).toEqual(["shadow-helper-x", "provider-b/model-b", "provider-c/model-c"]); + expect(response.status).toBe(400); + expect((await response.text()).includes("mandatory tool_choice")).toBe(true); + expect(isProxyError(response)).toBe(true); + }); + + test("a shadow-intercepted policy route keeps declared failover for upstream failures", async () => { + // The missing snapshot suppressed every retry on an intercepted policy route, not only the + // tool-choice refusal: an operator-approved failover target must still take the next + // eligible candidate after an ordinary retryable failure. + const trace = policyTrace(); + const config = { + ...fixtureConfig(), + shadowCallIntercept: { enabled: true, model: "policy/daily", sourceModels: ["shadow-helper"] }, + } as OcxConfig; + const logCtx = { requestedModel: "shadow-helper-x", routeDecision: trace, attempts: [] } as unknown as RequestLogContext; + const seenModels: string[] = []; + const req = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "shadow-helper-x", input: "hello", stream: false }), + }); + const response = await handleResponsesWithPolicyFallback(req, config, logCtx, {}, { + runCore: async (attempt, _config, ctx, options) => { + const body = await attempt.json() as { model: string }; + options.onRequestBodyParsed?.(body); + seenModels.push(body.model); + ctx.routeDecision = trace; + ctx.shadowCallRewrittenFrom = "shadow-helper"; + seedAttempt(ctx, "provider", body.model); + return seenModels.length === 1 + ? Response.json({ error: { type: "rate_limit_error" } }, { status: 429 }) + : Response.json({ status: "completed" }); + }, + }); + expect(response.status).toBe(200); + expect(seenModels).toEqual(["shadow-helper-x", "provider-b/model-b"]); + }); + + test("a shadow-source request without a policy decision takes no candidate retry", async () => { + // The widened snapshot only arms traversal; a source request that was not intercepted into + // a policy route still returns its first response untouched. + const config = { + ...fixtureConfig(), + shadowCallIntercept: { enabled: true, model: "provider-a/model-a", sourceModels: ["shadow-helper"] }, + } as OcxConfig; + const logCtx = { requestedModel: "shadow-helper-x", attempts: [] } as unknown as RequestLogContext; + let calls = 0; + const req = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "shadow-helper-x", input: "hello", stream: false }), + }); + const failure = Response.json({ error: { type: "rate_limit_error" } }, { status: 429 }); + const response = await handleResponsesWithPolicyFallback(req, config, logCtx, {}, { + runCore: async (attempt, _config, ctx, options) => { + calls += 1; + options.onRequestBodyParsed?.(await attempt.json()); + return failure; + }, + }); + expect(response).toBe(failure); + expect(calls).toBe(1); + }); }); diff --git a/tests/server/server-combo-required-tool-contract.test.ts b/tests/server/server-combo-required-tool-contract.test.ts index 35d57202d0d..607053d3737 100644 --- a/tests/server/server-combo-required-tool-contract.test.ts +++ b/tests/server/server-combo-required-tool-contract.test.ts @@ -256,6 +256,51 @@ describe("combo required-tool contract on buffered children (#7456)", () => { }); } + for (const [carrier, extra] of [ + ["tools", { tools: [{ type: "file_search" }], tool_choice: "required" }], + ["additional_tools", { + tools: [], tool_choice: "required", + input: [{ role: "user", content: "hi" }, { type: "additional_tools", tools: [{ type: "file_search" }] }], + }], + ["tool_search_output", { + tools: [], tool_choice: "required", + input: [{ role: "user", content: "hi" }, + { type: "tool_search_output", call_id: "ts_1", status: "completed", tools: [{ type: "file_search" }] }], + }], + ] as const) { + test(`parsed-catalog mandatory refusal hops the combo to a compatible child: ${carrier}`, async () => { + // The static eligibility check runs without the parsed catalog, so a hosted-only + // declaration that buildTools drops still looks preservable to it. Request preparation + // is the first point that sees the empty catalog — its refusal must keep the local + // refusal identity or the combo reports the child's bare 400 instead of hopping. + let chatSends = 0; + const bodies: unknown[] = []; + const backup = serve(async request => { + bodies.push(await request.json()); + return Response.json(completedWithHostedCall("m2", "file_search_call")); + }); + const chat = provider("openai-chat", "https://chat.fixture.invalid/v1", "key-a", { + fetch: (async () => { chatSends++; throw new Error("unexpected Chat send"); }) as typeof fetch }); + const response = await handleResponsesWith(comboConfig({ + a: chat, + b: provider("openai-responses", baseUrl(backup), "key-b"), + }), requiredRequest(extra)); + expect(response.status).toBe(200); + await response.text(); + expect(chatSends).toBe(0); + expect(bodies).toHaveLength(1); + }); + test(`parsed-catalog mandatory refusal reports combo exhaustion: ${carrier}`, async () => { + let sends = 0; + const chat = provider("openai-chat", "https://chat.fixture.invalid/v1", "key-a", { + fetch: (async () => { sends++; throw new Error("unexpected Chat send"); }) as typeof fetch }); + const response = await handleResponsesWith(comboConfig({ a: chat }), requiredRequest(extra)); + expect(response.status).toBe(503); + expect(await response.text()).toContain("attempted targets cannot preserve mandatory tool_choice"); + expect(sends).toBe(0); + }); + } + test.each(["ordinary", "exact-suffix", "disabled-fast"])("translated Chat child preserves forced tool_search on its actual wire: %s", async mode => { const bodies: unknown[] = []; const upstream = serve(async request => { From a369a4a3226aba5747e29dab68cb19de960d1d51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?SAL=C4=B0H=20=C3=96ZKARA?= Date: Sat, 7 Nov 2026 07:25:07 +0300 Subject: [PATCH 09/26] fix(responses): preserve late Devin signatures for DSH clients (#8230) * fix(responses): preserve late Devin signatures without index collisions * fix(responses): preserve signatures during failed stream cleanup * fix(responses): retain pending signatures on stall and rejected calls * fix(responses): keep late signatures beside non-leading thinking * docs(runtime): describe Devin turn delivery and Responses parsing * fix(responses): defer hidden carriers until tool closure * fix(responses): close text before hidden raw carriers --- .../src/content/docs/reference/adapters.md | 5 +- src/adapters/devin.ts | 12 +- src/bridge/response-json.ts | 14 +- src/bridge/sse.ts | 134 ++++- src/responses/parser.ts | 27 + src/responses/reasoning-envelope.ts | 4 + src/responses/schema.ts | 1 + src/types/request.ts | 3 +- structure/providers-and-adapters.md | 2 +- structure/transports/responses-wire-shapes.md | 2 +- .../bridge-raw-reasoning-hidden.test.ts | 60 ++ tests/adapters/bridge-stall.test.ts | 95 ++++ ...devin-anthropic-signature-fallback.test.ts | 2 +- .../devin-reasoning-continuation.test.ts | 221 ++++++++ tests/responses/reasoning-envelope.test.ts | 521 ++++++++++++++++++ 15 files changed, 1074 insertions(+), 29 deletions(-) diff --git a/docs-site/src/content/docs/reference/adapters.md b/docs-site/src/content/docs/reference/adapters.md index 6a9251504cb..4c147d1e37f 100644 --- a/docs-site/src/content/docs/reference/adapters.md +++ b/docs-site/src/content/docs/reference/adapters.md @@ -764,7 +764,10 @@ configuration that names the old id is rewritten at startup. This supports continuity but does not guarantee an upstream cache hit or a particular saving. - Reasoning continuity carries provider signatures across turns. If Cognition refuses a signed Anthropic replay before visible output, Devin retries once with the signature withheld and the - thinking text preserved. + thinking text preserved. Responses clients keep live text/tool streaming: a late turn signature + receives its own output index after the owning item closes, avoiding duplicate text and + unfinished-tool errors in index-keyed clients. Replay pairs the trailer only with adjacent + assistant output; matching IDs are checked when present, while ID-stripping clients remain supported. - Live model discovery via `GetCascadeModelConfigs`; the static seed is filtered against the account's live roster so models not on the plan drop out instead of failing at request time. - Tool definitions are encoded in the request and tool-call events are decoded from the response diff --git a/src/adapters/devin.ts b/src/adapters/devin.ts index 4c93aa8e452..5905a9fb2de 100644 --- a/src/adapters/devin.ts +++ b/src/adapters/devin.ts @@ -692,6 +692,11 @@ export function mapDevinToolCallStartForTests( return mapDevinToolCallStart(id, name, buildDevinReturnedToolNameMap(tools)); } +/** + * Create the native Devin adapter with bounded per-instance cascade-ID state. + * The configured provider ID selects the stored tenant slot; runTurn translates + * native chat events and marks turn signatures for downstream output ownership. + */ export function createDevinAdapter( provider: OcxProviderConfig, context: { providerId?: string } = {}, @@ -730,6 +735,11 @@ export function createDevinAdapter( }; }, + /** + * Execute one native chat turn through admitted sends and emit adapter events. + * Keeps signed continuation and optional unsigned retries request-scoped; late + * provider attestations are marked afterOutput for downstream replay ownership. + */ async runTurn(parsed: OcxParsedRequest, incoming: IncomingMeta, emit: (event: AdapterEvent) => void) { if (incoming.abortSignal?.aborted) { emit({ type: "error", message: DEVIN_CLIENT_CLOSED_MESSAGE, status: 499, retryable: false }); @@ -937,7 +947,7 @@ export function createDevinAdapter( if (event.kind === "reasoning_signature") { // Carried back out so the next turn can replay it in the prompt's // signature field; an unsigned replay is what the service ignores. - emit({ type: "thinking_signature", signature: encodeDevinSignature(event.signature, event.signatureType) }); + emit({ type: "thinking_signature", signature: encodeDevinSignature(event.signature, event.signatureType), afterOutput: true }); continue; } if (event.kind === "tool_call_start") { diff --git a/src/bridge/response-json.ts b/src/bridge/response-json.ts index bcdc51e7a4b..935c018aadf 100644 --- a/src/bridge/response-json.ts +++ b/src/bridge/response-json.ts @@ -217,6 +217,7 @@ function buildResponseJSONWithBudget( let rawReasoningForNextToolCall = ""; // Anthropic extended-thinking round-trip (batch): see bridgeToResponsesSSE counterpart. let batchSignature: string | undefined; + let batchSignatureAfter: string | undefined; let batchSignatureBytes = 0; let batchRedacted: string[] = []; let batchRedactedBytes = 0; @@ -276,12 +277,14 @@ function buildResponseJSONWithBudget( if (!summaryText && !batchSignature && batchRedacted.length === 0) return; const envelope: ReasoningEnvelope = {}; if (batchSignature) envelope.sig = batchSignature; + if (batchSignatureAfter) envelope.aft = batchSignatureAfter; if (batchRedacted.length > 0) envelope.red = batchRedacted; const hidden = options?.hideThinkingSummary === true; if (hidden && summaryText && (envelope.sig || envelope.red)) envelope.txt = summaryText; const encrypted = envelope.sig || envelope.red || envelope.txt ? encodeReasoningEnvelope(envelope, budget) : undefined; const sourceBytes = currentSummaryReasoning.bytes + batchSignatureBytes + batchRedactedBytes; batchSignature = undefined; + batchSignatureAfter = undefined; batchSignatureBytes = 0; batchRedacted = []; batchRedactedBytes = 0; @@ -440,12 +443,21 @@ function buildResponseJSONWithBudget( ); } break; - case "thinking_signature": + case "thinking_signature": { // Like streaming, retain the latest signature update until the next semantic // event. Flushing every update would manufacture signature-only siblings. batchSignatureBytes = replaceBatchRetainedString(batchSignatureBytes, e.signature, "reasoning"); batchSignature = e.signature; + const lastOutput = output[output.length - 1]; + // Open outputs still flush after their reasoning, preserving legacy ordering. + // An explicitly late signature for an already-closed output needs a back-reference. + batchSignatureAfter = e.afterOutput && !currentText.bytes && !currentToolCallId + && !currentSummaryReasoning.bytes && lastOutput + && (["message", "function_call", "custom_tool_call"].includes(lastOutput.type) + || (lastOutput.type === "tool_search_call" && lastOutput.execution === "client")) + ? lastOutput.id : undefined; break; + } case "redacted_thinking": flushText("commentary"); flushSummaryReasoning(); diff --git a/src/bridge/sse.ts b/src/bridge/sse.ts index 978dd29c3b0..671f40fbfd2 100644 --- a/src/bridge/sse.ts +++ b/src/bridge/sse.ts @@ -390,13 +390,16 @@ export function bridgeToResponsesSSE( // item as an ocxr1 encrypted_content envelope on close. hiddenThinkingText collects the // suppressed text under hideThinkingSummary so the signed text still round-trips. let pendingSignature: string | undefined; + let pendingSignatureAfter: string | undefined; + let pendingSignatureAfterOutput = false; let pendingSignatureBytes = 0; let pendingRedacted: string[] = []; let hiddenThinking = emptyChunks(); - const takeReasoningEnvelope = (hiddenText?: string): string | undefined => { + const takeReasoningEnvelope = (hiddenText?: string, after?: string): string | undefined => { if (!pendingSignature && pendingRedacted.length === 0) return undefined; const envelope: ReasoningEnvelope = {}; if (pendingSignature) envelope.sig = pendingSignature; + if (after ?? pendingSignatureAfter) envelope.aft = after ?? pendingSignatureAfter; if (pendingRedacted.length > 0) envelope.red = pendingRedacted; if (hiddenText) envelope.txt = hiddenText; const previousBytes = pendingSignatureBytes @@ -405,25 +408,84 @@ export function bridgeToResponsesSSE( const encoded = encodeReasoningEnvelope(envelope, budget); const reservation = budget?.reserveTransient(bytesOf(encoded), { kind: "reasoning" }); pendingSignature = undefined; + pendingSignatureAfter = undefined; + pendingSignatureAfterOutput = false; pendingSignatureBytes = 0; pendingRedacted = []; reservation?.commitRetained(); budget?.releaseRetained(previousBytes, { kind: "reasoning" }); return encoded; }; - // hideThinkingSummary path: no visible reasoning item exists, but a signed thinking block - // must still round-trip — emit an envelope-only reasoning item (empty summary, no text leak). - const flushHiddenReasoningEnvelope = () => { + let deferredHiddenReasoning: string[] = []; + let deferredHiddenReasoningIndex = 0; + const deferredReasoningEntryBytes = 32; // Queue bookkeeping, in addition to each retained payload. + /** Release unconsumed queue payloads and entry overhead on cancellation or translation overflow. */ + const discardDeferredHiddenReasoning = () => { + for (let i = deferredHiddenReasoningIndex; i < deferredHiddenReasoning.length; i++) { + budget?.releaseRetained(bytesOf(deferredHiddenReasoning[i]!) + deferredReasoningEntryBytes, { kind: "reasoning" }); + } + deferredHiddenReasoning = []; + deferredHiddenReasoningIndex = 0; + }; + /** Transfer a retained envelope into a closed item; release failed direct sends, while queues own theirs. */ + const emitHiddenReasoningEnvelope = (encrypted: string, queued = false): boolean => { + let transferred = false; + try { + if (closed) return false; + const itemId = outputItemId("reasoning", "rs"); + const item = { type: "reasoning", id: itemId, summary: [] as never[], encrypted_content: encrypted }; + emit("response.output_item.added", { output_index: outputIndex, item }); + if (closed) return false; + emit("response.output_item.done", { output_index: outputIndex, item }); + if (closed) return false; + retainFinishedItem(item as OutputItem, bytesOf(encrypted), "reasoning"); + transferred = true; + outputIndex++; + return true; + } finally { + if (!transferred && !queued) budget?.releaseRetained(bytesOf(encrypted), { kind: "reasoning" }); + } + }; + /** Drain independent hidden blocks only after the open tool has safely released its index. */ + const flushDeferredHiddenReasoning = () => { + if (currentToolCall) return; + while (deferredHiddenReasoningIndex < deferredHiddenReasoning.length) { + if (!emitHiddenReasoningEnvelope(deferredHiddenReasoning[deferredHiddenReasoningIndex]!, true)) { + discardDeferredHiddenReasoning(); + return; + } + budget?.releaseRetained(deferredReasoningEntryBytes, { kind: "reasoning" }); + deferredHiddenReasoning[deferredHiddenReasoningIndex++] = ""; + } + deferredHiddenReasoning = []; + deferredHiddenReasoningIndex = 0; + }; + /** Queue retained raw/ordinary envelopes behind an open tool, prioritizing a safely closed owner's trailer. */ + const deliverHiddenReasoningEnvelope = (encrypted: string, afterOutput = false) => { + if (currentToolCall) { + try { + budget?.chargeRetained(deferredReasoningEntryBytes, { kind: "reasoning" }); + deferredHiddenReasoning.push(encrypted); + } catch (error) { + budget?.releaseRetained(bytesOf(encrypted), { kind: "reasoning" }); + throw error; + } + return; + } + // A turn-level trailer must stay adjacent to its owner, ahead of ordinary hidden blocks. + if (afterOutput) emitHiddenReasoningEnvelope(encrypted); + flushDeferredHiddenReasoning(); + if (!afterOutput) emitHiddenReasoningEnvelope(encrypted); + }; + /** Seal hidden thinking without issuing an unfinished call; explicit late trailers wait for owner closure. */ + const flushHiddenReasoningEnvelope = (after?: string) => { + if (currentToolCall && pendingSignatureAfterOutput) return; const hiddenText = joinChunks(hiddenThinking); - const encrypted = takeReasoningEnvelope(hiddenText || undefined); + const afterOutput = pendingSignatureAfterOutput; + const encrypted = takeReasoningEnvelope(hiddenText || undefined, after); hiddenThinking = emptyChunks(); - if (!encrypted) return; - const itemId = outputItemId("reasoning", "rs"); - const item = { type: "reasoning", id: itemId, summary: [] as never[], encrypted_content: encrypted }; - emit("response.output_item.added", { output_index: outputIndex, item }); - emit("response.output_item.done", { output_index: outputIndex, item }); - retainFinishedItem(item as OutputItem, bytesOf(encrypted), "reasoning"); - outputIndex++; + if (encrypted) deliverHiddenReasoningEnvelope(encrypted, afterOutput); + else flushDeferredHiddenReasoning(); }; // hideThinkingSummary for RAW reasoning (openai-chat reasoning_content, kiro tags): no // visible reasoning item is emitted — the app renders nothing, so tool cells keep grouping @@ -447,12 +509,7 @@ export function bridgeToResponsesSSE( hiddenRawReasoning = emptyChunks(); reservation?.commitRetained(); budget?.releaseRetained(previousBytes, { kind: "reasoning" }); - const itemId = outputItemId("reasoning", "rs"); - const item = { type: "reasoning", id: itemId, summary: [] as never[], encrypted_content: encrypted }; - emit("response.output_item.added", { output_index: outputIndex, item }); - emit("response.output_item.done", { output_index: outputIndex, item }); - retainFinishedItem(item as OutputItem, bytesOf(encrypted), "reasoning"); - outputIndex++; + deliverHiddenReasoningEnvelope(encrypted); }; // Kiro reasoning round-trip. Kiro sends its encrypted blob at the END of a turn, while the // assistant message is still open, so this CANNOT emit on arrival: the open message still @@ -544,6 +601,7 @@ export function bridgeToResponsesSSE( retainFinishedItem(item as OutputItem, currentMsg.text.bytes + bytesOf(JSON.stringify(annotations))); outputIndex++; currentMsg = null; + if (pendingSignatureAfterOutput) flushHiddenReasoningEnvelope(item.id); }; const closeCurrentReasoning = () => { @@ -643,6 +701,8 @@ export function bridgeToResponsesSSE( budget?.closeCall(currentToolCall.callId); outputIndex++; currentToolCall = null; + if (pendingSignatureAfterOutput) flushHiddenReasoningEnvelope(item.id); + flushDeferredHiddenReasoning(); }; // Terminal-error / incomplete path for an open tool call (#765 remainder). @@ -683,6 +743,8 @@ export function bridgeToResponsesSSE( budget?.closeCall(currentToolCall.callId); outputIndex++; currentToolCall = null; + if (pendingSignatureAfterOutput) flushHiddenReasoningEnvelope(item.id); + flushDeferredHiddenReasoning(); }; const abortCurrentToolCallForTranslatorOverflow = () => { @@ -788,6 +850,7 @@ export function bridgeToResponsesSSE( const usage = (error as AdapterEvent | null | undefined)?.type === "error" ? (error as Extract).usage : undefined; if (usage) options?.onUsage?.(usage); + discardDeferredHiddenReasoning(); abortCurrentToolCallForTranslatorOverflow(); currentWebSearch = null; releasePendingWebSources(); @@ -884,9 +947,18 @@ export function bridgeToResponsesSSE( // Anthropic signature_delta supplies the latest signature, not an append-only // fragment (anthropic-sdk-typescript MessageStream). Keep consecutive updates // together; the next semantic event belongs to the following block. - if (pendingSignature !== undefined && event.type !== "thinking_signature" && event.type !== "heartbeat") { + if (pendingSignature !== undefined && event.type !== "thinking_signature" && event.type !== "heartbeat" + && !["done", "error", "incomplete", "tool_call_end"].includes(event.type)) { + // Terminal/tool-end handlers decide whether an open call completed safely. if (currentReasoning) closeCurrentReasoning(); - else flushHiddenReasoningEnvelope(); + else if (!pendingSignatureAfterOutput) { + // Hidden thinking cannot issue an unfinished tool. Seal each block independently + // and let the tool's safe close drain its deferred carriers at fresh indexes. + closeCurrentMessage("commentary"); + if (!options?.hideThinkingSummary) closeCurrentToolCall(); + flushHiddenReasoningEnvelope(); + } else if (!currentMsg && !currentToolCall) flushHiddenReasoningEnvelope(); + // Explicit late turn signatures wait for the open output's close helper. } switch (event.type) { case "assistant_boundary": { @@ -927,6 +999,7 @@ export function bridgeToResponsesSSE( } if (currentReasoning) closeCurrentReasoning(); if (currentRawReasoning) closeCurrentRawReasoning(); + if (currentMsg && hiddenRawReasoning.bytes > 0) closeCurrentMessage("commentary"); flushHiddenRawReasoning(); // Reasoning consumed by a REAL text turn, not a tool call: no cache target. // Empty text deltas must not wipe reasoning that precedes a tool call @@ -974,6 +1047,7 @@ export function bridgeToResponsesSSE( break; } case "thinking_delta": { + if (currentMsg) closeCurrentMessage("commentary"); if (options?.hideThinkingSummary) { // The hidden branch returns early, so flush any raw reasoning // that preceded the thinking block and clear the replay-cache @@ -988,7 +1062,6 @@ export function bridgeToResponsesSSE( ); break; } - if (currentMsg) closeCurrentMessage("commentary"); if (currentRawReasoning) closeCurrentRawReasoning(); flushHiddenRawReasoning(); if (event.thinking.length > 0) rawReasoningForNextToolCall = ""; @@ -1017,6 +1090,14 @@ export function bridgeToResponsesSSE( case "thinking_signature": { pendingSignatureBytes = replaceRetainedString(pendingSignatureBytes, event.signature, "reasoning"); pendingSignature = event.signature; + const lastOutput = finishedItems[finishedItems.length - 1]; + pendingSignatureAfterOutput = event.afterOutput === true; + pendingSignatureAfter = event.afterOutput + ? currentMsg?.itemId ?? currentToolCall?.itemId + ?? (!currentReasoning && lastOutput + && (["message", "function_call", "custom_tool_call"].includes(lastOutput.type) + || (lastOutput.type === "tool_search_call" && lastOutput.execution === "client")) ? lastOutput.id : undefined) + : undefined; // Delay closing until the next semantic event so a signature update cannot // create another block or become attached to the following thinking text. break; @@ -1026,7 +1107,7 @@ export function bridgeToResponsesSSE( if (currentReasoning) closeCurrentReasoning(); if (currentRawReasoning) closeCurrentRawReasoning(); flushHiddenRawReasoning(); - if (currentToolCall) closeCurrentToolCall(); + if (currentToolCall && !options?.hideThinkingSummary) closeCurrentToolCall(); budget?.chargeRetained(bytesOf(event.data), { kind: "reasoning" }); pendingRedacted.push(event.data); // A redacted block is complete at content_block_start. Emit it here, @@ -1202,6 +1283,7 @@ export function bridgeToResponsesSSE( && !toolCallArgumentsUsable(currentToolCall.args) ) { failCurrentToolCall(); + flushHiddenReasoningEnvelope(); const failure = responseError( 502, "upstream_error", @@ -1394,6 +1476,7 @@ export function bridgeToResponsesSSE( if (currentToolCall) failCurrentToolCall(); if (currentWebSearch) closeCurrentWebSearch("failed", []); releasePendingWebSources(); + flushHiddenReasoningEnvelope(); const failure = adapterFailureFromEvent(event); if (event.usage) options?.onUsage?.(event.usage); await awaitThoughtSignatureDurability(); @@ -1431,10 +1514,14 @@ export function bridgeToResponsesSSE( } if (!terminated) { if (!attemptTerminationCleanup(() => { + if (currentMsg) closeCurrentMessage(); + if (currentReasoning) closeCurrentReasoning(); + if (currentRawReasoning) closeCurrentRawReasoning(); flushHiddenRawReasoning(); if (currentToolCall) failCurrentToolCall(); if (currentWebSearch) closeCurrentWebSearch("failed", []); releasePendingWebSources(); + flushHiddenReasoningEnvelope(); })) return; const failure = responseError( 500, @@ -1473,6 +1560,7 @@ export function bridgeToResponsesSSE( if (currentToolCall) failCurrentToolCall(); if (currentWebSearch) closeCurrentWebSearch("failed", []); releasePendingWebSources(); + flushHiddenReasoningEnvelope(); })) return; options?.onUsage?.(undefined); await awaitThoughtSignatureDurability(); @@ -1528,6 +1616,7 @@ export function bridgeToResponsesSSE( if (currentToolCall) failCurrentToolCall(); if (currentWebSearch) closeCurrentWebSearch("failed", []); releasePendingWebSources(); + flushHiddenReasoningEnvelope(); })) return; // #1926 gap 2 residual: this beat callback is synchronous, so the durability // barrier is not awaited on the stall-timeout kill path. The in-memory store is @@ -1583,6 +1672,7 @@ export function bridgeToResponsesSSE( if (beat !== undefined) clearBeatInterval(beat); cancelUpstreamOnce(); releasePendingWebSources(); + discardDeferredHiddenReasoning(); disposeOwnedBudget(); }, })); diff --git a/src/responses/parser.ts b/src/responses/parser.ts index 575c2521cc9..2f9355eb1f4 100644 --- a/src/responses/parser.ts +++ b/src/responses/parser.ts @@ -217,6 +217,12 @@ function nameDestinationContent( return changed ? parts : content; } +/** + * Validate a Responses body and normalize its tools, options and conversation. + * Restores signed or opaque reasoning from replayed items; late signatures require + * adjacent output ownership and retain thinking/signature ordering. + * @throws Error when the body fails Responses schema validation. + */ export function parseRequest( body: unknown, parseOptions?: { replayCacheScope?: OcxReasoningReplayScopeRef }, @@ -460,6 +466,27 @@ export function parseRequest( continue; } + // Late turn signatures follow the closed output on the wire. Bind only to the + // immediately preceding item named by the carrier, never the following turn. + if (envelope?.aft) { + const previousItem = inputItems[itemIndex - 1] as { id?: string; type?: string; role?: string; execution?: string } | undefined; + const previous = messages[messages.length - 1]; + const properOutput = previousItem?.type === "function_call" || previousItem?.type === "custom_tool_call" + || (previousItem?.type === "tool_search_call" && previousItem.execution === "client") + || ((previousItem?.type === "message" || previousItem?.type === undefined) && previousItem?.role === "assistant"); + if (envelope.sig && properOutput && previous?.role === "assistant" + && (previousItem?.id === undefined || previousItem.id === envelope.aft)) { + // Preserve thinking/signature adjacency even after prefatory text or earlier calls. + const afterThinking = previous.content.findLastIndex(part => part.type === "thinking") + 1; + previous.content.splice(afterThinking, 0, { + type: "thinking", thinking: thinkingText, signature: envelope.sig, + ...(envelope.red ? { redacted: envelope.red } : {}), + ...(reasoning.id ? { itemId: reasoning.id } : {}), + }); + } + continue; + } + // Native/non-ocxr1 encrypted-only reasoning is opaque here. Do not create a detached // assistant turn or invent replayable plaintext/signatures from the encrypted payload. if (thinkingText.length > 0 || envelope?.sig || envelope?.red?.length) { diff --git a/src/responses/reasoning-envelope.ts b/src/responses/reasoning-envelope.ts index 628bd82838a..ecf719e6131 100644 --- a/src/responses/reasoning-envelope.ts +++ b/src/responses/reasoning-envelope.ts @@ -20,6 +20,8 @@ export const OCX_REASONING_PREFIX = "ocxr1:"; export interface ReasoningEnvelope { /** Anthropic thinking-block signature (signature_delta), if captured. */ sig?: string; + /** Immediately preceding output item owned by this late turn signature. */ + aft?: string; /** ID of the immediately following commentary copy of this signed thinking block. */ upd?: string; /** Raw redacted_thinking block data payloads, order preserved. */ @@ -147,6 +149,8 @@ export function decodeReasoningEnvelope(encryptedContent: string, budget?: Trans const obj = parsed as { sig?: unknown; red?: unknown }; const envelope: ReasoningEnvelope = {}; if (typeof obj.sig === "string") envelope.sig = obj.sig; + const aft = (parsed as { aft?: unknown }).aft; + if (typeof aft === "string" && aft.length > 0) envelope.aft = aft; const upd = (parsed as { upd?: unknown }).upd; if (typeof upd === "string" && /^msg_[a-f0-9]{32}$/.test(upd)) envelope.upd = upd; if (Array.isArray(obj.red)) { diff --git a/src/responses/schema.ts b/src/responses/schema.ts index 213e443d2d4..cb31f5677fd 100644 --- a/src/responses/schema.ts +++ b/src/responses/schema.ts @@ -63,6 +63,7 @@ const systemMessageItemSchema = z.object({ }); const assistantMessageItemSchema = z.object({ type: z.literal("message").optional(), + id: z.string().optional(), role: z.literal("assistant"), content: z.union([z.string(), z.array(outputContentBlockSchema)]).optional(), phase: z.enum(["commentary", "final_answer"]).optional(), diff --git a/src/types/request.ts b/src/types/request.ts index 1e5878e0b5f..cd6ddff7769 100644 --- a/src/types/request.ts +++ b/src/types/request.ts @@ -482,7 +482,8 @@ export type AdapterEvent = | { type: "thinking_delta"; thinking: string } // Anthropic extended-thinking round-trip: signature_delta for the current thinking block, and // opaque redacted_thinking blocks. Both must be replayed verbatim or tool-use turns 400. - | { type: "thinking_signature"; signature: string } + // afterOutput marks a turn-level attestation emitted after text/tools, not a new thinking block. + | { type: "thinking_signature"; signature: string; afterOutput?: true } | { type: "redacted_thinking"; data: string } // Kiro reasoning round-trip: the encrypted reasoning blob for the CURRENT assistant turn, tagged // with the wire field it arrived on. Never rendered — it only rides the reasoning item's envelope diff --git a/structure/providers-and-adapters.md b/structure/providers-and-adapters.md index f3306c1551e..c7718ea4d49 100644 --- a/structure/providers-and-adapters.md +++ b/structure/providers-and-adapters.md @@ -222,7 +222,7 @@ rewrite rules and the routed-id settlement. `src/adapters/devin/trajectory.ts` owns up to 256 process-local conversation trajectories, keyed by a digest of resolved credential identity, tenant host and named conversation. Own-thread identity takes precedence over session aliases. When a nonempty parent header accompanies an own-thread identity, the tagged parent/own pair identifies the conversation; standalone identities have a distinct tag, so a JSON-like caller string cannot alias a child tuple. Session-only keys retain their existing behavior, and a parent-only request does not borrow its parent's trajectory. Sequential turns reuse an inactive entry across request-scoped adapters; overlapping turns and a store full of active entries receive fresh unretained IDs. Only inactive entries are evicted, and a claim's idempotent release runs in the adapter's `finally` on success, error and cancellation. The optional trajectory travels through `src/adapters/devin/cloud-direct/chat.ts` to protobuf field #15.1; unnamed calls retain per-request allocation. A named claim remains held through the turn's existing reset/signature retries. This is identifier continuity, not proof of upstream cache hits. -Devin pairs a late signature only with immediately preceding unsigned thinking in one assistant message; a call between them breaks the pair. While a signed attempt is held for an optional unsigned retry, a timer emits plain heartbeats even when the upstream stalls. The held queue is capped at 1,024 events or approximately 1 MiB of UTF-16 reasoning, signature, and signature-type payload; signature types are individually capped at 4 KiB before UTF-8 decoding, and crossing either queue cap releases the events and disables that retry. Held usage frames merge per cumulative field, and the refused attempt's usage is added to the retry. A signed `invalid_argument` refusal is offered the unsigned retry before the history-overflow classifier sees any final refusal. If the send budget withholds that retry, the original refusal reaches the classifier and the recovery is recorded as withheld. +Devin marks its turn-level signature with `thinking_signature.afterOutput` in `src/types/request.ts`. The Responses SSE bridge closes the owning message/call before emitting a late signature carrier at a fresh index; client-executed `tool_search_call` outputs also own their late carriers, while absent/server execution does not. Its `aft` envelope field names the immediately preceding output item; `src/responses/schema.ts` preserves assistant message IDs, and `src/responses/parser.ts` reattaches only an adjacent matching trailer immediately after that assistant's last thinking part (or at the start when no thinking part exists), preserving thinking/signature adjacency even after prefatory text or earlier calls. Adjacent valid outputs may omit IDs for client replay compatibility; explicit ID mismatches, missing outputs and user/system boundaries drop the trailer rather than attach it to a later turn. Buffered JSON uses the same binding for an explicit late signature after a closed output. Ordinary block signatures, including hidden interleaved Anthropic blocks, retain their own exact text and signatures without backwards ownership. Hidden summary/raw thinking and redacted blocks do not complete an open tool call: independent carriers wait in a translator-budgeted queue until safe owner closure, with explicit turn trailers emitted adjacent to their owner before queued ordinary blocks. Queue payload and entry charges are released on cancellation, overflow or failed admission/emission; overflow and disconnect do not guarantee carrier delivery. Legacy unmarked carriers retain their ordering: a call between unsigned thinking and an unmarked signature breaks their pair. `tests/providers/devin-reasoning-continuation.test.ts` exercises SSE and terminal snapshot replay through the actual Devin prompt serializer; `tests/responses/reasoning-envelope.test.ts` enforces unique indexes, matching closes and exact delta ownership across successful, truncated and failed turns. Error/truncation handlers settle open calls as incomplete before flushing pending signatures; iterator EOF, exceptions and stall timeouts also retain admitted late carriers after safely closing their owners. Malformed-tool rejection retains pending normal block signatures without marking the rejected call complete. While a signed attempt is held for an optional unsigned retry, a timer emits plain heartbeats even when the upstream stalls. The held queue is capped at 1,024 events or approximately 1 MiB of UTF-16 reasoning, signature, and signature-type payload; signature types are individually capped at 4 KiB before UTF-8 decoding, and crossing either queue cap releases the events and disables that retry. Held usage frames merge per cumulative field, and the refused attempt's usage is added to the retry. A signed `invalid_argument` refusal is offered the unsigned retry before the history-overflow classifier sees any final refusal. If the send budget withholds that retry, the original refusal reaches the classifier and the recovery is recorded as withheld. Devin family selection in `src/adapters/devin/live-models.ts` counts the anchor's nonzero targets once and visits each candidate's own axes. Missing axes have order zero; extra nonzero axes, including prototype names such as `toString`, count as mismatches. Effort precedence and family confinement remain unchanged; `tests/providers/devin-family-resolution.test.ts` covers the sparse-scoring contract. diff --git a/structure/transports/responses-wire-shapes.md b/structure/transports/responses-wire-shapes.md index 45526777ae8..5f11fd37224 100644 --- a/structure/transports/responses-wire-shapes.md +++ b/structure/transports/responses-wire-shapes.md @@ -6,7 +6,7 @@ With a valid delay, `src/bridge/internal.ts` maps typed HTTP 429 `rate_limit_err ## Adapter incomplete reasons -`src/bridge/sse.ts`, `src/bridge/response-json.ts`, and `src/protocols/encoders/adapter-events.ts` normalize explicit incomplete reasons with `truncationReasonFor(reason) ?? reason`: known output-limit aliases become `max_output_tokens`, and refusal/filter aliases become `content_filter`. `src/responses/incomplete-reason.ts` applies the same classifier to native Responses JSON and SSE delivery in `src/server/responses/passthrough-delivery.ts` and to WebSocket frames in `src/server/responses/codex-ws-wire.ts`, including native control chains. It copies only changed `incomplete_details.reason` metadata, leaving output content and all other response fields intact. The compaction parser in `src/adapters/openai-responses/passthrough.ts` also normalizes its internal incomplete event. `tests/responses/incomplete-reason-normalization.test.ts` covers native HTTP delivery, WS terminal normalization and both adapter parser formats. `tool_call_end.incomplete` preserves unfinished function/custom items without argument-done frames in SSE, buffered JSON and direct encoders; it does not itself terminate the turn or complete those arguments. +`src/bridge/sse.ts`, `src/bridge/response-json.ts`, and `src/protocols/encoders/adapter-events.ts` normalize explicit incomplete reasons with `truncationReasonFor(reason) ?? reason`: known output-limit aliases become `max_output_tokens`, and refusal/filter aliases become `content_filter`. `src/responses/incomplete-reason.ts` applies the same classifier to native Responses JSON and SSE delivery in `src/server/responses/passthrough-delivery.ts` and to WebSocket frames in `src/server/responses/codex-ws-wire.ts`, including native control chains. It copies only changed `incomplete_details.reason` metadata, leaving output content and all other response fields intact. The compaction parser in `src/adapters/openai-responses/passthrough.ts` also normalizes its internal incomplete event. `tests/responses/incomplete-reason-normalization.test.ts` covers native HTTP delivery, WS terminal normalization and both adapter parser formats. `tool_call_end.incomplete` preserves unfinished function/custom items without argument-done frames in SSE, buffered JSON and direct encoders; it does not itself terminate the turn or complete those arguments. `src/bridge/sse.ts` closes an open message/call before emitting a late signature carrier at a fresh index; terminal output positions match added/done indexes, while truncated/error calls remain incomplete. Its `aft` carrier binds replay to the adjacent preceding output ID under the [Devin reasoning contract](../providers-and-adapters.md), preserving thinking/signature adjacency after prefatory output without changing ordinary block-signature ordering. `tests/responses/reasoning-envelope.test.ts` covers visible/hidden reasoning, success/truncation/error terminals, safe tool status before pending-signature flush, carrier retention on iterator EOF/throw/stall, malformed and undeclared tool rejection, and exact index-keyed delta consumption. Hidden thinking/redacted events do not issue unfinished calls; independent signed blocks are budgeted and deferred until safe tool closure, preserving exact payloads and fresh indexes. When buffered hidden raw reasoning precedes resumed text, the current message closes before the carrier is emitted; consecutive text deltas without buffered raw reasoning stay in one item. Cancellation/overflow discard undeliverable queue ownership rather than continuing emission after close. Unknown reasons remain verbatim, including `adapter_eof`, `max_messages`, and `steered`; message, retryability, usage, and incomplete status are preserved. `tests/adapters/bridge-nonstreaming-terminal.test.ts` covers both Responses projections and real passthrough parsing; the direct Chat and Messages parity suites cover the client-wire outcomes. The Chat writer does not depend on that upstream pass alone: `chatCompletionsIncompleteOutcome` in `src/chat/outbound.ts` classifies through the same `truncationReasonFor` table for the SSE converter (and the collector that folds it), the direct encoder in `src/protocols/encoders/chat.ts` and `responsesJsonToChatCompletion`, so a raw alias such as Volcengine Ark's `length` still ends with `finish_reason: "length"` (or `content_filter`). Reasons outside the table, including `adapter_eof`, `upstream_stall_timeout`, `missing_terminal_event`, `max_messages` and `steered`, remain Chat errors. `tests/responses/chat-incomplete-length-reason.test.ts` covers the converter outcomes and the handler's streamed and buffered wire. ## Compaction image input diff --git a/tests/adapters/bridge-raw-reasoning-hidden.test.ts b/tests/adapters/bridge-raw-reasoning-hidden.test.ts index e33b4f37e8a..862ce983b21 100644 --- a/tests/adapters/bridge-raw-reasoning-hidden.test.ts +++ b/tests/adapters/bridge-raw-reasoning-hidden.test.ts @@ -47,6 +47,66 @@ const REPLAY_SCOPE: OcxReasoningReplayScopeRef = { const GLOBAL_SCOPE: OcxReasoningReplayScopeRef = { ...REPLAY_SCOPE, clientThreadId: "global" }; const sseOpts = (hide: boolean) => ({ hideThinkingSummary: hide, replayCacheScope: REPLAY_SCOPE }); +describe("hidden raw reasoning between text chunks", () => { + for (const options of [{ hideThinkingSummary: true }, { hideRawReasoning: true }]) { + const mode = "hideThinkingSummary" in options ? "summary-hidden" : "raw-hidden"; + for (const failed of [false, true]) { + test(`${mode} separates message indexes around hidden raw, failed=${failed}`, async () => { + const frames = await collectSse(bridgeToResponsesSSE(replay([ + { type: "text_delta", text: "before" }, + { type: "reasoning_raw_delta", text: "private raw" }, + { type: "text_delta", text: "after" }, + failed ? { type: "error", code: "upstream_error", message: "synthetic failure" } : { type: "done" }, + ]), "m", undefined, undefined, undefined, undefined, undefined, options)); + const added = frames.filter(frame => frame.event === "response.output_item.added").map(frame => frame.data); + const done = frames.filter(frame => frame.event === "response.output_item.done").map(frame => frame.data); + expect(added.map(event => event.output_index)).toEqual([0, 1, 2]); + expect(done.map(event => event.output_index)).toEqual([0, 1, 2]); + const items = done.map(event => event.item as Record); + expect(items.map(item => item.type)).toEqual(["message", "reasoning", "message"]); + expect(decodeReasoningEnvelope(items[1]!.encrypted_content as string)).toEqual({ txt: "private raw" }); + const slots = new Map(); + for (const frame of frames) { + const event = frame.data; + const index = event.output_index as number; + if (frame.event === "response.output_item.added") slots.set(index, { id: (event.item as Record).id, text: "" }); + if (frame.event === "response.output_text.delta") { + expect(slots.get(index)?.id).toBe(event.item_id); + slots.get(index)!.text += event.delta as string; + } + if (frame.event === "response.output_item.done") { + const item = event.item as Record; + expect(slots.get(index)?.id).toBe(item.id); + if (item.type === "message") { + expect(slots.get(index)?.text).toBe(index === 0 ? "before" : "after"); + expect(item.content).toEqual([{ type: "output_text", text: index === 0 ? "before" : "after", annotations: [] }]); + } + slots.delete(index); + } + } + expect(slots.size).toBe(0); + expect(items[0]!.phase).toBe("commentary"); + expect(frames.filter(frame => frame.event === "response.reasoning_text.delta")).toHaveLength(0); + expect(JSON.stringify(frames)).not.toContain("private raw"); + const terminal = frames.filter(frame => frame.event === (failed ? "response.failed" : "response.completed")); + expect(terminal).toHaveLength(1); + expect((terminal[0]!.data.response as Record).output).toEqual(items); + }); + } + test(`${mode} keeps consecutive plain text chunks in one item`, async () => { + const frames = await collectSse(bridgeToResponsesSSE(replay([ + { type: "text_delta", text: "before" }, { type: "text_delta", text: "after" }, { type: "done" }, + ]), "m", undefined, undefined, undefined, undefined, undefined, options)); + const added = frames.filter(frame => frame.event === "response.output_item.added"); + expect(added).toHaveLength(1); + const item = frames.find(frame => frame.event === "response.output_item.done")!.data.item as Record; + expect(item.content).toEqual([{ type: "output_text", text: "beforeafter", annotations: [] }]); + expect(frames.filter(frame => frame.event === "response.output_text.delta").map(frame => [frame.data.output_index, frame.data.item_id, frame.data.delta])) + .toEqual([[0, item.id, "before"], [0, item.id, "after"]]); + }); + } +}); + describe("hidden raw reasoning (hideThinkingSummary parity for reasoning_raw_delta)", () => { beforeEach(() => { clearReasoningReplayCacheForTests(); diff --git a/tests/adapters/bridge-stall.test.ts b/tests/adapters/bridge-stall.test.ts index 85a063c3334..906e3a08978 100644 --- a/tests/adapters/bridge-stall.test.ts +++ b/tests/adapters/bridge-stall.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { bridgeToResponsesSSE } from "../../src/bridge"; import type { AdapterEvent } from "../../src/types"; +import { decodeReasoningEnvelope } from "../../src/responses/reasoning-envelope"; async function collectSse(stream: ReadableStream): Promise<{ event?: string; data: Record }[]> { const reader = stream.getReader(); @@ -49,6 +50,100 @@ const flush = async () => { }; describe("bridge stall watchdog budget", () => { + for (const hidden of [false, true]) { + for (const earlyEnd of [false, true]) { + test(`stall preserves late signature, hidden=${hidden}, earlyEnd=${earlyEnd}`, async () => { + const { timers, tick } = beatSeam(); + const stalled = Promise.withResolvers(); + const release = Promise.withResolvers(); + let cancelled = 0; + async function* upstream(): AsyncGenerator { + yield { type: "thinking_delta", thinking: "signed thought" }; + yield { type: "tool_call_start", id: "call_stall", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + if (earlyEnd) yield { type: "tool_call_end" }; + yield { type: "thinking_signature", signature: "stall-sig", afterOutput: true }; + stalled.resolve(); + await release.promise; + } + const framesPromise = collectSse(bridgeToResponsesSSE( + upstream(), "model", undefined, undefined, undefined, + () => { cancelled++; release.resolve(); }, 500, + { stallTimeoutSec: 1, hideThinkingSummary: hidden, timers }, + )); + try { + await stalled.promise; + // Clear the last upstream activity, then exhaust the two-tick stall budget. + tick(); tick(); tick(); + const frames = await framesPromise; + const added = frames.filter(frame => frame.event === "response.output_item.added").map(frame => frame.data); + const done = frames.filter(frame => frame.event === "response.output_item.done").map(frame => frame.data); + expect(added.map(event => event.output_index)).toEqual(added.map((_, index) => index)); + expect(done.map(event => event.output_index)).toEqual(added.map(event => event.output_index)); + const items = done.map(event => event.item as Record); + const tool = items.find(item => item.type === "function_call")!; + expect(tool.status).toBe(earlyEnd ? "completed" : "incomplete"); + const carriers = items.filter(item => typeof item.encrypted_content === "string"); + expect(carriers).toHaveLength(1); + expect(decodeReasoningEnvelope(carriers[0]!.encrypted_content as string)).toEqual({ + sig: "stall-sig", aft: tool.id, ...(hidden ? { txt: "signed thought" } : {}), + }); + const terminal = frames.filter(frame => frame.event === "response.incomplete"); + expect(terminal).toHaveLength(1); + expect(stallReason(terminal[0]!)).toBe("upstream_stall_timeout"); + expect((terminal[0]!.data.response as Record).output).toEqual(items); + expect(frames.some(frame => frame.event === "response.completed" || frame.event === "response.failed")).toBe(false); + expect(cancelled).toBe(1); + } finally { release.resolve(); } + }); + } + } + + for (const mode of ["normal", "multiple", "afterOutput"] as const) { + test(`hidden thinking does not issue a stalled open tool, mode=${mode}`, async () => { + const { timers, tick } = beatSeam(); + const stalled = Promise.withResolvers(); + const release = Promise.withResolvers(); + const blocks = mode === "multiple" ? 2 : 1; + /** Interleave signed hidden blocks into an argument stream that subsequently stalls. */ + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_hidden_stall", name: "read" }; + yield { type: "tool_call_delta", arguments: '{"value":' }; + for (let i = 0; i < blocks; i++) { + yield { type: "thinking_delta", thinking: `hidden-${i}` }; + yield { type: "thinking_signature", signature: `sig-${i}`, ...(mode === "afterOutput" ? { afterOutput: true } : {}) }; + if (i === 0) yield { type: "tool_call_delta", arguments: '"ok"' }; + } + stalled.resolve(); + await release.promise; + } + const framesPromise = collectSse(bridgeToResponsesSSE( + upstream(), "model", undefined, undefined, undefined, () => release.resolve(), 500, + { stallTimeoutSec: 1, hideThinkingSummary: true, timers }, + )); + try { + await stalled.promise; + tick(); tick(); tick(); + const frames = await framesPromise; + const added = frames.filter(frame => frame.event === "response.output_item.added").map(frame => frame.data); + const done = frames.filter(frame => frame.event === "response.output_item.done").map(frame => frame.data); + const items = done.map(event => event.item as Record); + const tool = items.find(item => item.type === "function_call")!; + expect(tool.status).toBe("incomplete"); + expect(tool.arguments).toBe('{"value":"ok"'); + expect(frames.filter(frame => frame.event === "response.function_call_arguments.done")).toHaveLength(0); + expect(added.map(event => event.output_index)).toEqual(added.map((_, index) => index)); + expect(done.map(event => event.output_index)).toEqual(added.map(event => event.output_index)); + expect(items.filter(item => item.encrypted_content).map(item => decodeReasoningEnvelope(item.encrypted_content as string))) + .toEqual(Array.from({ length: blocks }, (_, i) => ({ sig: `sig-${i}`, txt: `hidden-${i}`, ...(mode === "afterOutput" ? { aft: tool.id } : {}) }))); + const terminal = frames.filter(frame => frame.event === "response.incomplete"); + expect(terminal).toHaveLength(1); + expect(stallReason(terminal[0]!)).toBe("upstream_stall_timeout"); + expect((terminal[0]!.data.response as Record).output).toEqual(items); + } finally { release.resolve(); } + }); + } + test("a disabled budget (stallTimeoutSec: 0) does not kill a silent upstream on the first beat", async () => { // Regression: resolveStallTimeoutSec(0) used to floor to 1s, and maxStallTicks then rounded to // 0, so `++stallTicks >= 0` tripped the watchdog on the very first silent beat — killing a diff --git a/tests/providers/devin-anthropic-signature-fallback.test.ts b/tests/providers/devin-anthropic-signature-fallback.test.ts index edbdf846330..36e1abfa6b3 100644 --- a/tests/providers/devin-anthropic-signature-fallback.test.ts +++ b/tests/providers/devin-anthropic-signature-fallback.test.ts @@ -183,7 +183,7 @@ describe("Devin Anthropic signature fallback", () => { expect(requests).toHaveLength(1); const kinds = events.map(e => e.type); expect(kinds).toContain("thinking_delta"); - expect(events).toContainEqual({ type: "thinking_signature", signature: encodeDevinSignature("EpcBNew", "anthropic") }); + expect(events).toContainEqual({ type: "thinking_signature", signature: encodeDevinSignature("EpcBNew", "anthropic"), afterOutput: true }); expect(kinds.indexOf("thinking_delta")).toBeLessThan(kinds.indexOf("text_delta")); }); diff --git a/tests/providers/devin-reasoning-continuation.test.ts b/tests/providers/devin-reasoning-continuation.test.ts index 62c04fea57c..3fed20f7f66 100644 --- a/tests/providers/devin-reasoning-continuation.test.ts +++ b/tests/providers/devin-reasoning-continuation.test.ts @@ -5,6 +5,8 @@ import { encodeString, iterFields } from "../../src/adapters/devin/cloud-direct/ import { decodeDevinSignature, encodeDevinSignature, hasAnthropicSignature } from "../../src/adapters/devin/reasoning-signature"; import { encodeReasoningEnvelope } from "../../src/responses/reasoning-envelope"; import { parseRequest } from "../../src/responses/parser"; +import { bridgeToResponsesSSE, buildResponseJSON } from "../../src/bridge"; +import type { AdapterEvent } from "../../src/types"; // Shapes measured live on GetChatMessage: swe-2-high streams reasoning, then the // visible answer, then one frame carrying #10 delta_signature and #21 @@ -94,6 +96,225 @@ describe("Devin reasoning continuation across turns", () => { expect(wire.get(18)).toBe("sealed"); }); + for (const thinking of [false, true]) { + for (const tool of [false, true]) { + for (const earlyEnd of [false, true]) { + test(`streamed late signature replays with thinking=${thinking}, tool=${tool}, earlyEnd=${earlyEnd}`, async () => { + async function* upstream(): AsyncGenerator { + if (thinking) yield { type: "thinking_delta", thinking: "pick 482916" }; + if (tool) { + yield { type: "tool_call_start", id: "call_1", name: "get_time" }; + yield { type: "tool_call_delta", arguments: "{}" }; + if (earlyEnd) yield { type: "tool_call_end" }; + } else yield { type: "text_delta", text: "The answer" }; + yield { type: "thinking_signature", signature: encodeDevinSignature(SEALED, "sealed"), afterOutput: true }; + if (tool && !earlyEnd) yield { type: "tool_call_end" }; + yield { type: "done", usage: { inputTokens: 1, outputTokens: 1 } }; + } + const adapterEvents = await Array.fromAsync(upstream()); + const buffered = buildResponseJSON(adapterEvents, "swe-2"); + const bufferedOutput = buffered.output as any[]; + const outputs: any[][] = [bufferedOutput, bufferedOutput.map(({ id: _id, ...item }: any) => item)]; + for (const hidden of [false, true]) { + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "swe-2", undefined, undefined, undefined, undefined, 2000, + { hideThinkingSummary: hidden }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const terminal = events.find(event => event.type === "response.completed").response; + const liveItems = events.filter(event => event.type === "response.output_item.done").map(event => event.item); + for (const output of [liveItems, terminal.output]) { + outputs.push(output, output.map(({ id: _id, ...item }: any) => item)); + } + } + for (const output of outputs) { + const history = mapOcxMessagesToDevin(parseRequest({ + model: "devin/swe-2", input: [ + { role: "user", content: "go" }, ...output, + ...(tool ? [{ type: "function_call_output", call_id: "call_1", output: "12:00" }] : []), + { role: "user", content: "continue" }, + ], + })); + const assistant = history.find(message => message.role === "assistant"); + expect(assistant?.signature, JSON.stringify(output)).toBe(SEALED); + expect(assistant?.signature_type).toBe("sealed"); + expect(assistant?.thinking).toBe(thinking ? "pick 482916" : undefined); + const prompt = assistantPrompt(history); + expect(prompt.get(12)).toBe(SEALED); + expect(prompt.get(18)).toBe("sealed"); + } + }); + } + } + } + + test.each(["text", "tool"])("late signature stays beside non-leading thinking after %s", async prefix => { + for (const hidden of [false, true]) { + for (const earlyEnd of [false, true]) { + async function* upstream(): AsyncGenerator { + if (prefix === "text") yield { type: "text_delta", text: "I will inspect this." }; + else { + yield { type: "tool_call_start", id: "call_earlier", name: "list" }; + yield { type: "tool_call_delta", arguments: "{}" }; + yield { type: "tool_call_end" }; + } + yield { type: "thinking_delta", thinking: "non-leading thought" }; + yield { type: "tool_call_start", id: "call_owner", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + if (earlyEnd) yield { type: "tool_call_end" }; + yield { type: "thinking_signature", signature: encodeDevinSignature(SEALED, "sealed"), afterOutput: true }; + if (!earlyEnd) yield { type: "tool_call_end" }; + yield { type: "done", usage: { inputTokens: 1, outputTokens: 1 } }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "swe-2", undefined, undefined, undefined, undefined, 2000, + { hideThinkingSummary: hidden }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const live = events.filter(event => event.type === "response.output_item.done").map(event => event.item); + const snapshot = events.find(event => event.type === "response.completed").response.output; + for (const output of [live, snapshot]) { + for (const idless of [false, true]) { + const parsed = parseRequest({ model: "devin/swe-2", input: [ + { role: "user", content: "go" }, + ...output.map(({ id, ...item }: any) => idless ? item : { ...item, id }), + ] }); + const history = mapOcxMessagesToDevin(parsed); + const assistant = history.find(message => message.role === "assistant"); + expect(assistant?.thinking).toBe("non-leading thought"); + expect(assistant?.signature).toBe(SEALED); + expect(assistant?.signature_type).toBe("sealed"); + const prompt = assistantPrompt(history); + expect(prompt.get(11)).toBe("non-leading thought"); + expect(prompt.get(12)).toBe(SEALED); + expect(prompt.get(18)).toBe("sealed"); + if (!hidden) { + const parts = parsed.context.messages.find(message => message.role === "assistant")!.content; + expect(parts.map(part => part.type)).toEqual([prefix === "text" ? "text" : "toolCall", "thinking", "thinking", "toolCall"]); + } + } + } + } + } + }); + + test.each([false, true])("late adjacency keeps multiple thinking blocks conservative, earlierSigned=%s", earlierSigned => { + const signature = encodeDevinSignature(SEALED, "sealed"); + const parsed = parseRequest({ model: "devin/swe-2", input: [ + { type: "message", role: "assistant", content: "preface" }, + { type: "reasoning", summary: [{ type: "summary_text", text: "first" }], + ...(earlierSigned ? { encrypted_content: encodeReasoningEnvelope({ sig: "earlier-signature" }) } : {}) }, + { type: "function_call", call_id: "call_earlier", name: "list", arguments: "{}" }, + { type: "reasoning", summary: [{ type: "summary_text", text: "second" }] }, + { type: "function_call", id: "fc_owner", call_id: "call_owner", name: "read", arguments: "{}" }, + { type: "reasoning", summary: [], encrypted_content: encodeReasoningEnvelope({ sig: signature, aft: "fc_owner", red: ["opaque"] }) }, + ] }); + const parts = parsed.context.messages[0]!.content; + expect(parts.map(part => part.type)).toEqual(["text", "thinking", "toolCall", "thinking", "thinking", "toolCall"]); + expect(parts[4]).toMatchObject({ type: "thinking", thinking: "", signature, redacted: ["opaque"] }); + const history = mapOcxMessagesToDevin(parsed); + expect(history[0]?.signature).toBeUndefined(); + expect(assistantPrompt(history).has(12)).toBe(false); + }); + + test.each(["wrong", "user-boundary", "system-boundary", "typed-system-boundary"])("a trailer with a %s owner cannot sign the next turn", variant => { + const history = mapOcxMessagesToDevin(parseRequest({ + model: "devin/swe-2", input: [ + { role: "user", content: "go" }, + { type: "reasoning", summary: [{ type: "summary_text", text: "earlier thought" }] }, + { type: "message", role: "assistant", id: "msg_owner", content: "answer" }, + ...(variant === "user-boundary" ? [{ role: "user", content: "boundary" }] : []), + ...(variant === "system-boundary" ? [{ role: "system", content: "boundary" }] : []), + ...(variant === "typed-system-boundary" ? [{ type: "message", role: "system", id: "msg_owner", content: "boundary" }] : []), + { type: "reasoning", summary: [], encrypted_content: encodeReasoningEnvelope({ + sig: encodeDevinSignature(SEALED, "sealed"), aft: variant === "wrong" ? "msg_other" : "msg_owner", + }) }, + { role: "assistant", content: "next answer" }, + ], + })); + expect(history.filter(message => message.role === "assistant").every(message => !message.signature)).toBe(true); + }); + + for (const earlyEnd of [false, true]) { + test(`late signature survives client tool search, earlyEnd=${earlyEnd}`, async () => { + async function* upstream(): AsyncGenerator { + yield { type: "thinking_delta", thinking: "find a tool" }; + yield { type: "tool_call_start", id: "call_search", name: "tool_search" }; + yield { type: "tool_call_delta", arguments: "{}" }; + if (earlyEnd) yield { type: "tool_call_end" }; + yield { type: "thinking_signature", signature: encodeDevinSignature(SEALED, "sealed"), afterOutput: true }; + if (!earlyEnd) yield { type: "tool_call_end" }; + yield { type: "done", usage: { inputTokens: 1, outputTokens: 1 } }; + } + const buffered = buildResponseJSON(await Array.fromAsync(upstream()), "swe-2", { toolSearchToolNames: new Set(["tool_search"]) }); + const outputs: any[][] = [buffered.output as any[]]; + for (const hidden of [false, true]) { + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "swe-2", undefined, undefined, new Set(["tool_search"]), undefined, 2000, + { hideThinkingSummary: hidden }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + outputs.push(events.filter(event => event.type === "response.output_item.done").map(event => event.item)); + outputs.push(events.find(event => event.type === "response.completed").response.output); + } + for (const output of outputs) { + expect(output.find(item => item.type === "tool_search_call")).toMatchObject({ call_id: "call_search", execution: "client" }); + for (const idless of [false, true]) { + const parsed = parseRequest({ model: "devin/swe-2", input: output.map(({ id, ...item }: any) => idless ? item : { ...item, id }) }); + const history = mapOcxMessagesToDevin(parsed); + expect(history[0]?.signature).toBe(SEALED); + expect(history[0]?.thinking).toBe("find a tool"); + expect(parsed.context.messages[0]?.content).toContainEqual({ type: "toolCall", id: "call_search", name: "tool_search", arguments: {} }); + expect(assistantPrompt(history).get(12)).toBe(SEALED); + } + } + }); + } + + test.each([undefined, "server"])("tool search with execution=%s cannot own a late signature", execution => { + const parsed = parseRequest({ model: "devin/swe-2", input: [ + { type: "message", role: "assistant", content: "earlier answer" }, + { type: "tool_search_call", id: "tsc_owner", call_id: "call_search", execution, arguments: {} }, + { type: "reasoning", summary: [], encrypted_content: encodeReasoningEnvelope({ sig: encodeDevinSignature(SEALED, "sealed"), aft: "tsc_owner" }) }, + { type: "message", role: "assistant", content: "next answer" }, + ] }); + expect(mapOcxMessagesToDevin(parsed).every(message => !message.signature)).toBe(true); + }); + + for (const custom of [false, true]) { + test(`late signature binds parallel ${custom ? "custom" : "function"} calls and preserves redacted data`, async () => { + async function* upstream(): AsyncGenerator { + yield { type: "thinking_delta", thinking: "parallel thought" }; + for (const id of ["call_a", "call_b"]) { + yield { type: "tool_call_start", id, name: "read" }; + yield { type: "tool_call_delta", arguments: custom ? "raw input" : "{}" }; + yield { type: "tool_call_end" }; + } + yield { type: "thinking_signature", signature: encodeDevinSignature(SEALED, "sealed"), afterOutput: true }; + yield { type: "done", usage: { inputTokens: 1, outputTokens: 1 } }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "swe-2", undefined, custom ? new Set(["read"]) : undefined, undefined, undefined, 2000, + { hideThinkingSummary: true }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const output = events.find(event => event.type === "response.completed").response.output; + expect(output.slice(0, 2).map((item: any) => item.type)).toEqual(Array(2).fill(custom ? "custom_tool_call" : "function_call")); + for (const idless of [false, true]) { + const parsed = parseRequest({ model: "devin/swe-2", input: output.map(({ id, ...item }: any) => idless ? item : { ...item, id }) }); + const history = mapOcxMessagesToDevin(parsed); + expect(history[0]?.signature).toBe(SEALED); + expect(history[0]?.thinking).toBe("parallel thought"); + expect(parsed.context.messages[0]?.content.filter(part => part.type === "toolCall")).toHaveLength(2); + } + const redacted = parseRequest({ model: "m", input: [ + { type: "message", role: "assistant", content: "answer" }, + { type: "reasoning", summary: [], encrypted_content: encodeReasoningEnvelope({ sig: "sig", aft: "msg_owner", red: ["opaque"] }) }, + ] }); + expect(redacted.context.messages[0]?.content[0]).toMatchObject({ type: "thinking", signature: "sig", redacted: ["opaque"] }); + }); + } + test("a signature separated from its text by a call is not paired", () => { const history = mapOcxMessagesToDevin(parseRequest({ model: "devin/swe-2", diff --git a/tests/responses/reasoning-envelope.test.ts b/tests/responses/reasoning-envelope.test.ts index 6ffc2d750f8..fb497a3e7ed 100644 --- a/tests/responses/reasoning-envelope.test.ts +++ b/tests/responses/reasoning-envelope.test.ts @@ -83,6 +83,527 @@ describe("reasoning and tool/result envelopes", () => { }); }); +describe("late reasoning signatures keep Responses item indexes unique", () => { + const usage = { inputTokens: 1, outputTokens: 1 }; + const endings: AdapterEvent[] = [ + { type: "done", usage }, + { type: "done", usage, stopReason: "max_tokens" }, + { type: "incomplete", reason: "adapter_eof", usage }, + { type: "error", code: "upstream_error", message: "Synthetic failure" }, + ]; + for (const hidden of [false, true]) { + for (const tool of [false, true]) { + for (const [index, ending] of endings.entries()) { + test(`${hidden ? "hidden" : "visible"} thinking, ${tool ? "tool" : "text"}, terminal ${index}`, async () => { + async function* upstream(): AsyncGenerator { + yield { type: "thinking_delta", thinking: "A synthetic summary" }; + if (tool) { + yield { type: "tool_call_start", id: "call_fixture", name: "glob" }; + yield { type: "tool_call_delta", arguments: '{"pattern":"*.py"}' }; + } else { + yield { type: "text_delta", text: "Selam!" }; + } + // Devin seals the reasoning after answer/tool deltas, not after its summary. + yield { type: "thinking_signature", signature: "synthetic-old-signature", afterOutput: true }; + yield { type: "heartbeat" }; + yield { type: "thinking_signature", signature: "synthetic-latest-signature", afterOutput: true }; + yield ending; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "swe-2", undefined, undefined, undefined, undefined, 2000, + { hideThinkingSummary: hidden }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const added = events.filter(event => event.type === "response.output_item.added"); + const done = events.filter(event => event.type === "response.output_item.done"); + // A client keyed by output_index must never lose an open block to another ID. + expect(new Set(added.map(event => event.output_index)).size).toBe(added.length); + expect(added.map(event => event.output_index)).toEqual(added.map((_, i) => i)); + for (const event of added) { + const closed = done.filter(end => end.item.id === event.item.id); + expect(closed).toHaveLength(1); + expect(closed[0].output_index).toBe(event.output_index); + } + // Model the index-keyed consumer used by pi-ai/DSH. Reusing an index + // overwrites its open slot and leaves duplicate text or an unfinished call. + const slots = new Map(); + const deltas = new Map(); + const completed: string[] = []; + for (const event of events) { + if (event.type === "response.output_item.added") slots.set(event.output_index, event.item.id); + if (["response.output_text.delta", "response.function_call_arguments.delta", "response.reasoning_summary_text.delta"].includes(event.type)) { + expect(slots.get(event.output_index)).toBe(event.item_id); + deltas.set(event.output_index, (deltas.get(event.output_index) ?? "") + event.delta); + } + if (event.type === "response.output_item.done") { + expect(slots.get(event.output_index)).toBe(event.item.id); + if (event.item.type === "message") expect(deltas.get(event.output_index)).toBe("Selam!"); + if (event.item.type === "function_call") expect(deltas.get(event.output_index)).toBe('{"pattern":"*.py"}'); + if (event.item.type === "reasoning" && !hidden && !event.item.encrypted_content) { + expect(deltas.get(event.output_index)).toBe("A synthetic summary"); + } + completed.push(event.item.id); + slots.delete(event.output_index); + } + } + expect(slots.size).toBe(0); + expect(completed).toHaveLength(added.length); + const terminal = events.find(event => ["response.completed", "response.incomplete", "response.failed"].includes(event.type)); + expect(terminal.response.output.map((item: { id: string }) => item.id)).toEqual(added.map(event => event.item.id)); + const carriers = done.filter(event => event.item.encrypted_content); + expect(carriers).toHaveLength(1); + expect(decodeReasoningEnvelope(carriers[0].item.encrypted_content)?.sig).toBe("synthetic-latest-signature"); + const output = done.filter(event => event.item.type === (tool ? "function_call" : "message")); + expect(output).toHaveLength(1); + if (tool) { + expect(output[0].item.arguments).toBe('{"pattern":"*.py"}'); + expect(output[0].item.status).toBe(index === 0 ? "completed" : "incomplete"); + } else { + expect(output[0].item.content[0].text).toBe("Selam!"); + if (index === 0) expect(output[0].item.phase).toBe("final_answer"); + } + }); + } + } + } +}); + +describe("pending signature terminal safety", () => { + const usage = { inputTokens: 1, outputTokens: 1 }; + const endings: AdapterEvent[] = [ + { type: "error", code: "upstream_error", message: "synthetic failure" }, + { type: "incomplete", reason: "adapter_eof", usage }, + { type: "done", usage, stopReason: "max_tokens" }, + { type: "tool_call_end", incomplete: true }, + { type: "done", usage }, + ]; + for (const afterOutput of [false, true]) { + for (const [index, ending] of endings.entries()) { + test(`open tool signature afterOutput=${afterOutput}, ending=${index}`, async () => { + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_terminal", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + yield { type: "thinking_signature", signature: "terminal-sig", ...(afterOutput ? { afterOutput: true } : {}) }; + yield ending; + } + const wire = await new Response(bridgeToResponsesSSE(upstream(), "m")).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const tool = events.find(event => event.type === "response.output_item.done" && event.item.type === "function_call"); + expect(tool.item.status).toBe(index === 4 ? "completed" : "incomplete"); + expect(events.filter(event => event.type === "response.function_call_arguments.done")).toHaveLength(index === 4 ? 1 : 0); + const carrier = events.find(event => event.type === "response.output_item.done" && event.item.encrypted_content); + expect(decodeReasoningEnvelope(carrier.item.encrypted_content)).toEqual({ sig: "terminal-sig", ...(afterOutput ? { aft: tool.item.id } : {}) }); + }); + } + } + for (const afterOutput of [false, true]) { + test(`malformed tool termination preserves signature afterOutput=${afterOutput}`, async () => { + /** Emit an invalid call followed by a provider signature and explicit call end. */ + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_malformed", name: "read" }; + yield { type: "tool_call_delta", arguments: "{" }; + yield { type: "thinking_signature", signature: "malformed-sig", ...(afterOutput ? { afterOutput: true } : {}) }; + yield { type: "tool_call_end" }; + yield { type: "done", usage }; + } + const wire = await new Response(bridgeToResponsesSSE(upstream(), "m")).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const done = events.filter(event => event.type === "response.output_item.done"); + const tool = done.find(event => event.item.type === "function_call"); + const carrier = done.find(event => event.item.encrypted_content); + expect(tool.item.status).toBe("incomplete"); + expect(events.filter(event => event.type === "response.function_call_arguments.done")).toHaveLength(0); + expect(decodeReasoningEnvelope(carrier?.item.encrypted_content ?? "")).toEqual({ sig: "malformed-sig", ...(afterOutput ? { aft: tool.item.id } : {}) }); + expect(events.find(event => event.type === "response.failed").response.output).toEqual(done.map(event => event.item)); + expect(events.filter(event => event.type === "response.completed")).toHaveLength(0); + }); + } + for (const afterOutput of [false, true]) { + test(`undeclared tool termination preserves signature afterOutput=${afterOutput}`, async () => { + /** Retain a completed allowed call while rejecting a later undeclared call. */ + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_allowed", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + yield { type: "tool_call_end" }; + yield { type: "thinking_signature", signature: "undeclared-sig", ...(afterOutput ? { afterOutput: true } : {}) }; + yield { type: "tool_call_start", id: "call_forbidden", name: "undeclared" }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, + { declaredToolNames: new Set(["read"]), enforceDeclaredToolNames: true }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const done = events.filter(event => event.type === "response.output_item.done"); + const tool = done.find(event => event.item.type === "function_call"); + const carrier = done.find(event => event.item.encrypted_content); + expect(tool.item.status).toBe("completed"); + expect(done.filter(event => event.item.type === "function_call").map(event => event.item.name)).toEqual(["read"]); + expect(decodeReasoningEnvelope(carrier?.item.encrypted_content ?? "")).toEqual({ sig: "undeclared-sig", ...(afterOutput ? { aft: tool.item.id } : {}) }); + expect(events.find(event => event.type === "response.failed").response.output).toEqual(done.map(event => event.item)); + expect(events.filter(event => event.type === "response.completed")).toHaveLength(0); + }); + } + for (const ending of ["eof", "throw"] as const) { + for (const earlyEnd of [false, true]) { + test(`late tool signature survives ${ending}, earlyEnd=${earlyEnd}`, async () => { + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_abnormal", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + if (earlyEnd) yield { type: "tool_call_end" }; + yield { type: "thinking_signature", signature: "abnormal-sig", afterOutput: true }; + if (ending === "throw") throw new Error("synthetic failure"); + } + const wire = await new Response(bridgeToResponsesSSE(upstream(), "m")).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const done = events.filter(event => event.type === "response.output_item.done"); + expect(done.map(event => event.output_index)).toEqual([0, 1]); + expect(done[0].item.status).toBe(earlyEnd ? "completed" : "incomplete"); + expect(decodeReasoningEnvelope(done[1].item.encrypted_content)).toEqual({ sig: "abnormal-sig", aft: done[0].item.id }); + const terminal = events.find(event => event.type === `response.${ending === "throw" ? "failed" : "incomplete"}`); + expect(terminal.response.output).toEqual(done.map(event => event.item)); + expect(events.filter(event => event.type === "response.completed")).toHaveLength(0); + }); + } + } +}); + +describe("hidden thinking while a tool remains open", () => { + const endings = ["error", "incomplete", "truncated", "eof", "throw", "malformed", "tool-incomplete", "tool-end", "done"] as const; + for (const blocks of [1, 2]) { + for (const ending of endings) { + test(`${blocks} ordinary hidden blocks preserve open call through ${ending}`, async () => { + await checkHiddenTool(blocks, ending, false); + }); + } + } + for (const ending of endings) { + test(`afterOutput hidden block preserves open call through ${ending}`, async () => { + await checkHiddenTool(1, ending, true); + }); + } + test("redacted hidden blocks do not issue an unfinished call or merge signed blocks", async () => { + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_redacted", name: "read" }; + yield { type: "tool_call_delta", arguments: "{" }; + yield { type: "thinking_delta", thinking: "exact thought" }; + yield { type: "thinking_signature", signature: "normal-sig" }; + yield { type: "redacted_thinking", data: "opaque" }; + yield { type: "error", code: "upstream_error", message: "synthetic failure" }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, { hideThinkingSummary: true }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const items = events.filter(event => event.type === "response.output_item.done").map(event => event.item); + expect(items[0].status).toBe("incomplete"); + expect(items.filter(item => item.encrypted_content).map(item => decodeReasoningEnvelope(item.encrypted_content))) + .toEqual([{ sig: "normal-sig", txt: "exact thought" }, { red: ["opaque"] }]); + expect(events.filter(event => event.type === "response.function_call_arguments.done")).toHaveLength(0); + }); + + test("unsigned hidden thinking cannot issue an unfinished call", async () => { + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_unsigned", name: "read" }; + yield { type: "tool_call_delta", arguments: "{" }; + yield { type: "thinking_delta", thinking: "unsealed" }; + yield { type: "error", code: "upstream_error", message: "synthetic failure" }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, { hideThinkingSummary: true }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + expect(events.find(event => event.type === "response.output_item.done").item.status).toBe("incomplete"); + expect(events.filter(event => event.type === "response.function_call_arguments.done")).toHaveLength(0); + }); + + test("deferred hidden blocks stay bounded by the translator budget", async () => { + const budget = createTranslatorBudget({ maxTurnBytes: 4096 }); + let produced = 0; + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_bounded", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + for (; produced < 1000; produced++) { + yield { type: "thinking_delta", thinking: "x".repeat(256) }; + yield { type: "thinking_signature", signature: `sig-${produced}` }; + yield { type: "tool_call_delta", arguments: "" }; + } + yield { type: "done" }; + } + try { + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, + { hideThinkingSummary: true, translatorBudget: budget }, + )).text(); + expect(produced).toBeLessThan(1000); + expect(budget.snapshot().highWaterBytes).toBeLessThanOrEqual(4096); + expect(wire).toContain('"code":"translation_buffer_limit"'); + expect(wire).not.toContain("event: response.function_call_arguments.done"); + expect(wire).not.toContain('"status":"completed"'); + } finally { budget.dispose(); } + expect(budget.snapshot().currentBytes).toBe(0); + }); + + test.each(["entry", "direct"] as const)("borrowed budget releases payload after %s admission failure", async stage => { + const budget = createTranslatorBudget({ maxTurnBytes: 65536 }); + const charge = budget.chargeRetained.bind(budget); + const reserve = budget.reserveTransient.bind(budget); + let failDirectFrame = false; + const chargeSpy = spyOn(budget, "chargeRetained").mockImplementation((bytes, scope) => { + if (stage === "entry" && scope.kind === "reasoning" && bytes === 32) throw new TranslatorBudgetExceededError("reasoning", 65536); + charge(bytes, scope); + }); + const reserveSpy = spyOn(budget, "reserveTransient").mockImplementation((bytes, scope) => { + if (failDirectFrame && scope.kind === "live_transient") { + failDirectFrame = false; + throw new TranslatorBudgetExceededError("live_transient", 65536); + } + return reserve(bytes, scope); + }); + async function* upstream(): AsyncGenerator { + if (stage === "entry") { + yield { type: "tool_call_start", id: "call_admission", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + } + yield { type: "thinking_delta", thinking: "hidden" }; + yield { type: "thinking_signature", signature: "sig" }; + if (stage === "entry") yield { type: "tool_call_delta", arguments: "" }; + yield { type: "done" }; + } + try { + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, + { hideThinkingSummary: true, translatorBudget: budget, outputItemIds: { + next(kind) { if (stage === "direct" && kind === "reasoning") failDirectFrame = true; return `${kind}_admission`; }, + } }, + )).text(); + expect(wire.match(/event: response.failed/g)).toHaveLength(1); + expect(wire).toContain('"code":"translation_buffer_limit"'); + expect(wire).not.toContain("event: response.function_call_arguments.done"); + expect(budget.snapshot().currentBytes).toBe(0); + } finally { chargeSpy.mockRestore(); reserveSpy.mockRestore(); budget.dispose(); } + }); + + test.each([2048, 4096, 8192])("deferred drain respects budget=%s without a second terminal", async maxTurnBytes => { + const budget = createTranslatorBudget({ maxTurnBytes }); + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_drain", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + for (let i = 0; i < 3; i++) { + yield { type: "thinking_delta", thinking: "x".repeat(256) }; + yield { type: "thinking_signature", signature: `sig-${i}` }; + yield { type: "tool_call_delta", arguments: "" }; + } + yield { type: "error", code: "upstream_error", message: "synthetic failure" }; + } + try { + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, + { hideThinkingSummary: true, translatorBudget: budget }, + )).text(); + expect(wire.match(/event: response.failed/g)).toHaveLength(1); + expect(wire.match(/data: \[DONE\]/g)).toHaveLength(1); + expect(wire).not.toContain("event: response.function_call_arguments.done"); + expect(wire).not.toContain('"status":"completed"'); + expect(budget.snapshot().highWaterBytes).toBeLessThanOrEqual(maxTurnBytes); + } finally { budget.dispose(); } + expect(budget.snapshot().currentBytes).toBe(0); + }); + + test.each([false, true])("late trailer stays adjacent ahead of deferred normal blocks, failed=%s", async failed => { + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_mixed", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + yield { type: "thinking_delta", thinking: "normal" }; + yield { type: "thinking_signature", signature: "normal-sig" }; + yield { type: "tool_call_delta", arguments: "" }; + yield { type: "thinking_delta", thinking: "late" }; + yield { type: "thinking_signature", signature: "late-sig", afterOutput: true }; + if (failed) yield { type: "error", code: "upstream_error", message: "synthetic failure" }; + else yield { type: "done" }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, { hideThinkingSummary: true }, + )).text(); + const items = wire.split("\n").filter(line => line.startsWith("data: {")) + .map(line => JSON.parse(line.slice(6))).filter(event => event.type === "response.output_item.done").map(event => event.item); + expect(items[0].status).toBe(failed ? "incomplete" : "completed"); + expect(items.slice(1).map(item => decodeReasoningEnvelope(item.encrypted_content))).toEqual([ + { sig: "late-sig", txt: "late", aft: items[0].id }, { sig: "normal-sig", txt: "normal" }, + ]); + }); + + for (const next of ["summary", "redacted"] as const) { + for (const failed of [false, true]) { + test(`hidden raw followed by ${next} keeps the open tool index, failed=${failed}`, async () => { + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_raw", name: "read" }; + yield { type: "tool_call_delta", arguments: '{"value":' }; + yield { type: "reasoning_raw_delta", text: "raw text" }; + if (next === "summary") { + yield { type: "thinking_delta", thinking: "summary text" }; + yield { type: "thinking_signature", signature: "summary-sig" }; + } else yield { type: "redacted_thinking", data: "opaque" }; + yield { type: "tool_call_delta", arguments: '"ok"}' }; + if (failed) yield { type: "error", code: "upstream_error", message: "synthetic failure" }; + else { yield { type: "tool_call_end" }; yield { type: "done" }; } + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, { hideThinkingSummary: true }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const added = events.filter(event => event.type === "response.output_item.added"); + const done = events.filter(event => event.type === "response.output_item.done"); + expect(added.map(event => event.output_index)).toEqual([0, 1, 2]); + expect(done.map(event => event.output_index)).toEqual([0, 1, 2]); + expect(done[0].item).toMatchObject({ type: "function_call", arguments: '{"value":"ok"}', status: failed ? "incomplete" : "completed" }); + expect(events.filter(event => event.type === "response.function_call_arguments.delta").map(event => [event.output_index, event.item_id, event.delta])) + .toEqual([[0, done[0].item.id, '{"value":'], [0, done[0].item.id, '"ok"}']]); + expect(done.slice(1).map(event => decodeReasoningEnvelope(event.item.encrypted_content))).toEqual([ + { txt: "raw text" }, next === "summary" ? { sig: "summary-sig", txt: "summary text" } : { red: ["opaque"] }, + ]); + expect(events.find(event => event.type === (failed ? "response.failed" : "response.completed")).response.output).toEqual(done.map(event => event.item)); + }); + } + } + + test.each([false, true])("redacted block cannot consume a late trailer before owner close, failed=%s", async failed => { + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_late_red", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + yield { type: "thinking_delta", thinking: "normal" }; + yield { type: "thinking_signature", signature: "normal-sig" }; + yield { type: "tool_call_delta", arguments: "" }; + yield { type: "thinking_delta", thinking: "late" }; + yield { type: "thinking_signature", signature: "late-sig", afterOutput: true }; + yield { type: "redacted_thinking", data: "opaque" }; + yield { type: "tool_call_delta", arguments: "" }; + if (failed) yield { type: "error", code: "upstream_error", message: "synthetic failure" }; + else { yield { type: "tool_call_end" }; yield { type: "done" }; } + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, { hideThinkingSummary: true }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const done = events.filter(event => event.type === "response.output_item.done"); + expect(done.map(event => event.output_index)).toEqual([0, 1, 2]); + expect(done[0].item.status).toBe(failed ? "incomplete" : "completed"); + expect(done.slice(1).map(event => decodeReasoningEnvelope(event.item.encrypted_content))).toEqual([ + { sig: "late-sig", txt: "late", aft: done[0].item.id, red: ["opaque"] }, { sig: "normal-sig", txt: "normal" }, + ]); + expect(events.find(event => event.type === (failed ? "response.failed" : "response.completed")).response.output).toEqual(done.map(event => event.item)); + }); + + test.each(["signed", "raw"] as const)("client cancellation releases deferred %s payload and queue charges", async mode => { + const budget = createTranslatorBudget({ maxTurnBytes: 65536 }); + const queued = Promise.withResolvers(); + const release = Promise.withResolvers(); + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_cancel", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + for (let i = 0; i < 2; i++) { + if (mode === "signed") { + yield { type: "thinking_delta", thinking: `hidden-${i}` }; + yield { type: "thinking_signature", signature: `sig-${i}` }; + } else { + yield { type: "reasoning_raw_delta", text: `hidden-${i}` }; + yield { type: "thinking_delta", thinking: "" }; + } + yield { type: "tool_call_delta", arguments: "" }; + } + queued.resolve(); + await release.promise; + } + const stream = bridgeToResponsesSSE(upstream(), "m", undefined, undefined, undefined, + () => release.resolve(), 2000, { hideThinkingSummary: true, translatorBudget: budget }); + const reader = stream.getReader(); + const draining = (async () => { while (!(await reader.read()).done) {} })(); + try { + await queued.promise; + const before = budget.snapshot().currentBytes; + const encodedBytes = [0, 1].reduce((sum, i) => sum + Buffer.byteLength(encodeReasoningEnvelope({ txt: `hidden-${i}`, ...(mode === "signed" ? { sig: `sig-${i}` } : {}) })), 0); + await reader.cancel(); + await draining; + expect(before - budget.snapshot().currentBytes).toBe(encodedBytes + 2 * 32); + } finally { release.resolve(); budget.dispose(); } + }); + + /** Exercise hidden blocks between argument fragments without allowing thinking to issue a call. */ + async function checkHiddenTool(blocks: number, ending: typeof endings[number], afterOutput: boolean) { + const usage = { inputTokens: 1, outputTokens: 1 }; + async function* upstream(): AsyncGenerator { + yield { type: "tool_call_start", id: "call_interleaved", name: "read" }; + yield { type: "tool_call_delta", arguments: '{"value":' }; + for (let i = 0; i < blocks; i++) { + yield { type: "thinking_delta", thinking: `hidden-${i}` }; + yield { type: "thinking_signature", signature: `sig-${i}`, ...(afterOutput ? { afterOutput: true } : {}) }; + if (i === 0) yield { type: "tool_call_delta", arguments: '"ok"' }; + } + if (ending !== "malformed") yield { type: "tool_call_delta", arguments: "}" }; + if (ending === "throw") throw new Error("synthetic interleaved failure"); + if (ending === "error") yield { type: "error", code: "upstream_error", message: "synthetic interleaved failure" }; + if (ending === "incomplete") yield { type: "incomplete", reason: "adapter_eof", usage }; + if (ending === "truncated") yield { type: "done", usage, stopReason: "max_tokens" }; + if (ending === "malformed" || ending === "tool-end") yield { type: "tool_call_end" }; + if (ending === "tool-incomplete") yield { type: "tool_call_end", incomplete: true }; + if (ending === "tool-end" || ending === "done") yield { type: "done", usage }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, { hideThinkingSummary: true }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const added = events.filter(event => event.type === "response.output_item.added"); + const done = events.filter(event => event.type === "response.output_item.done"); + const tool = done.find(event => event.item.type === "function_call"); + const success = ending === "tool-end" || ending === "done"; + expect(tool.item.status).toBe(success ? "completed" : "incomplete"); + expect(tool.item.arguments).toBe(ending === "malformed" ? '{"value":"ok"' : '{"value":"ok"}'); + expect(events.filter(event => event.type === "response.function_call_arguments.done")).toHaveLength(success ? 1 : 0); + expect(added.map(event => event.output_index)).toEqual(added.map((_, index) => index)); + expect(done.map(event => event.output_index)).toEqual(added.map(event => event.output_index)); + const envelopes = done.filter(event => event.item.encrypted_content).map(event => decodeReasoningEnvelope(event.item.encrypted_content)); + expect(envelopes).toEqual(Array.from({ length: blocks }, (_, i) => ({ sig: `sig-${i}`, txt: `hidden-${i}`, ...(afterOutput ? { aft: tool.item.id } : {}) }))); + expect(events.filter(event => event.type === "response.reasoning_summary_text.delta")).toHaveLength(0); + const terminal = events.filter(event => ["response.completed", "response.failed", "response.incomplete"].includes(event.type)); + expect(terminal).toHaveLength(1); + expect(terminal[0].response.output).toEqual(done.map(event => event.item)); + } +}); + +describe("normal interleaved thinking signatures", () => { + for (const hidden of [false, true]) { + test(`each block retains its exact text and signature, hidden=${hidden}`, async () => { + async function* upstream(): AsyncGenerator { + yield { type: "text_delta", text: "before" }; + yield { type: "thinking_delta", thinking: "thinking A" }; + yield { type: "thinking_signature", signature: "sig-A" }; + yield { type: "text_delta", text: "between" }; + yield { type: "thinking_delta", thinking: "thinking B" }; + yield { type: "thinking_signature", signature: "sig-B" }; + yield { type: "tool_call_start", id: "normal_call", name: "read" }; + yield { type: "tool_call_delta", arguments: "{}" }; + yield { type: "done", usage: { inputTokens: 1, outputTokens: 1 } }; + } + const wire = await new Response(bridgeToResponsesSSE( + upstream(), "m", undefined, undefined, undefined, undefined, 2000, + { hideThinkingSummary: hidden }, + )).text(); + const events = wire.split("\n").filter(line => line.startsWith("data: {")).map(line => JSON.parse(line.slice(6))); + const added = events.filter(event => event.type === "response.output_item.added"); + const done = events.filter(event => event.type === "response.output_item.done"); + expect(added.map(event => event.output_index)).toEqual(added.map((_, i) => i)); + expect(done.map(event => event.output_index)).toEqual(added.map(event => event.output_index)); + const buffered = buildResponseJSON(await Array.fromAsync(upstream()), "m", { hideThinkingSummary: hidden }); + for (const items of [done.map(event => event.item), buffered.output as any[]]) { + expect(items.map(item => item.type)).toEqual(["message", "reasoning", "message", "reasoning", "function_call"]); + const envelopes = items.filter(item => item.type === "reasoning").map(item => decodeReasoningEnvelope(item.encrypted_content)); + expect(envelopes).toEqual(hidden + ? [{ sig: "sig-A", txt: "thinking A" }, { sig: "sig-B", txt: "thinking B" }] + : [{ sig: "sig-A" }, { sig: "sig-B" }]); + } + }); + } +}); + describe("reasoning allocation admission", () => { test.each(["ascii", "\"\\\n\u0000", "한글😀", "\ud800", "\udc00", ""])('sizes JSON strings exactly: %j', value => { const data = { sig: value, red: [value, ""], txt: value, krc: value, omitted: undefined }; From 803d6c0b4dbb4823a9379e485e417f3f9e3d976c Mon Sep 17 00:00:00 2001 From: Epinephrine Date: Sat, 7 Nov 2026 13:31:22 +0900 Subject: [PATCH 10/26] fix(anthropic): batch affinity writes and confirm operator resets (#8020) Batch Anthropic affinity writes and confirm operator resets. Co-authored-by: Epinephrine --- .../fr/reference/cli/providers-accounts.md | 5 +- .../docs/fr/reference/management-api.md | 7 +- .../ja/reference/cli/providers-accounts.md | 5 +- .../docs/ja/reference/management-api.md | 7 +- .../ko/reference/cli/providers-accounts.md | 5 +- .../docs/ko/reference/management-api.md | 7 +- .../docs/reference/cli/providers-accounts.md | 5 +- .../content/docs/reference/management-api.md | 7 +- .../ru/reference/cli/providers-accounts.md | 5 +- .../docs/ru/reference/management-api.md | 7 +- .../tr/reference/cli/providers-accounts.md | 5 +- .../docs/tr/reference/management-api.md | 7 +- .../zh-cn/reference/cli/providers-accounts.md | 5 +- .../docs/zh-cn/reference/management-api.md | 7 +- .../zh-tw/reference/cli/providers-accounts.md | 5 +- .../docs/zh-tw/reference/management-api.md | 7 +- .../provider-workspace/ProviderAuthPanel.tsx | 8 + .../components/provider-workspace/types.ts | 2 + gui/src/hooks/useProviderAccountPools.ts | 177 +++++- gui/src/i18n/de.ts | 3 + gui/src/i18n/en.ts | 3 + gui/src/i18n/fr.ts | 3 + gui/src/i18n/ja.ts | 3 + gui/src/i18n/ko.ts | 3 + gui/src/i18n/pt.ts | 3 + gui/src/i18n/ru.ts | 3 + gui/src/i18n/tr.ts | 3 + gui/src/i18n/vi.ts | 3 + gui/src/i18n/zh-TW.ts | 3 + gui/src/i18n/zh.ts | 3 + gui/src/pages/Providers.tsx | 2 + gui/src/provider-affinity-reset-retries.ts | 83 +++ gui/tests/auth-panel-refresh-render.test.tsx | 30 +- .../provider-account-pause-refresh.test.tsx | 568 +++++++++++++++++- .../provider-affinity-reset-retries.test.ts | 170 ++++++ scripts/test-layout/layout-clients.json | 1 + .../ocx/references/01_management_surface.md | 10 +- skills/ocx/references/01_surface_accounts.md | 20 +- src/cli/account-affinity-reset.ts | 42 ++ src/cli/account.ts | 5 + src/cli/capabilities-accounts.ts | 8 + src/oauth/anthropic-routing.ts | 49 +- src/oauth/anthropic-session-affinity-store.ts | 33 +- src/server/management/oauth-account-routes.ts | 73 ++- src/server/management/route-registry.ts | 1 + src/server/responses/request-transport.ts | 4 + structure/cli-management.md | 2 + structure/gui-and-management-api.md | 12 +- structure/providers/anthropic-account-pool.md | 13 +- structure/remote-link.md | 2 +- .../anthropic-affinity-restart.test.ts | 241 +++++++- .../anthropic-affinity-store.test.ts | 52 +- .../anthropic-quota-dispatch.test.ts | 92 ++- .../claude-endpoint-lifetime.test.ts | 107 ++++ .../claude-messages-endpoint.test.ts | 94 +-- tests/cli/cli-account-pool-verbs.test.ts | 44 ++ tests/cli/cli-link-child.test.ts | 26 + tests/fixtures/test-layout-expected.json | 1 + tests/helpers/claude-endpoint-lifetime.ts | 133 ++++ .../local-account-switch-capability.test.ts | 2 +- 60 files changed, 2069 insertions(+), 167 deletions(-) create mode 100644 gui/src/provider-affinity-reset-retries.ts create mode 100644 gui/tests/provider-affinity-reset-retries.test.ts create mode 100644 src/cli/account-affinity-reset.ts create mode 100644 tests/claude-integration/claude-endpoint-lifetime.test.ts create mode 100644 tests/helpers/claude-endpoint-lifetime.ts diff --git a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md index 432161e84fe..2aeeaf1a3d1 100644 --- a/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/fr/reference/cli/providers-accounts.md @@ -129,7 +129,7 @@ Répertoriez et changez de compte de fournisseur et de pools de clés API via le la surface est : ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex account pool, OAuth accounts and API keys (identifiers shown masked as the API returns them). history openai [--limit <1-200>] Recent routing decisions for one Codex pool account. @@ -143,6 +143,7 @@ pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. clear-cooldown Drop a cooldown the proxy set after an upstream failure. +reset-affinity [--json] Confirm all session bindings are cleared, even with no accounts; selection and pause state stay unchanged. Run on the Home; a Remote Link Child names the Home and sends no request. strategy [] Stratégie du pool ; least-loaded est réservé à Kiro. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -160,7 +161,7 @@ A selection-order change applies from the next unbound request and never moves a Les sous-commandes nécessitent que le proxy soit en cours d'exécution et résolvent automatiquement son port d'exécution enregistré, sauf `import-orca` : l'aperçu est purement local, et `import-orca --apply` exige que le proxy soit arrêté. Les opérations réussies se terminent 0. Utilisation invalide, un fournisseur ou un identifiant inconnu, un -proxy, ou un échec API se termine 1. Les champs d'informations d'identification sont affichés exactement comme la gestion API +proxy inaccessible ou un échec API renvoient normalement le code 1. Pour `reset-affinity`, HTTP 404 renvoie 4 et HTTP 409 renvoie 5 ; les autres échecs API et les résultats non confirmés renvoient 1. Les champs d'informations d'identification sont affichés exactement comme la gestion API les renvoie (y compris son masquage) ; Les clés API brutes et les jetons OAuth ne sont jamais restitués. Affichage les commodités sont synthétisées côté client, comme dans le tableau de bord : `main` désigne la connexion Codex App du groupe de comptes `openai`, les comptes OAuth sans adresse e-mail apparaissent sous la forme `Account N`, diff --git a/docs-site/src/content/docs/fr/reference/management-api.md b/docs-site/src/content/docs/fr/reference/management-api.md index 25f81a99073..3bf84f5a3b2 100644 --- a/docs-site/src/content/docs/fr/reference/management-api.md +++ b/docs-site/src/content/docs/fr/reference/management-api.md @@ -279,15 +279,18 @@ Tant qu’une liste initiale fiable n’est pas disponible, les requêtes PUT va | `GET /api/oauth/status` | Sonder le flux OAuth d'un fournisseur | 400 fournisseur inconnu | | `POST /api/oauth/logout` | Supprimer les informations d'identification du fournisseur sélectionné | 400 fournisseur inconnu ; `oauth_mutation_busy` | | `GET, DELETE /api/oauth/accounts` | Répertorier les comptes masqués ou supprimer un compte Les lignes Kiro ajoutent `autoSelectable` et un `skipReason` fermé en cas d’exclusion de la sélection automatique ; un compte actif unique peut encore servir. Le quota reste facultatif. | 400 invalide provider/id ; 404 compte manquant ; `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` | Sélectionnez le compte OAuth actif | 400 invalide provider/account ; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` | Sélectionnez le compte OAuth actif | 400 invalide provider/account ; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; sélection enregistrée ; renvoyer la même requête même si le compte est actif) | | `GET, PUT, PATCH /api/oauth/accounts/pool` | Lire ou mettre à jour la stratégie du pool OAuth Anthropic | 400 fournisseur non Anthropic ou stratégie invalide | | `POST /api/oauth/accounts/clear-cooldown` | Effacer le temps de recharge d'un compte OAuth | 400 invalide provider/account | | `PUT /api/oauth/accounts/alias` | Définir ou supprimer un alias de compte OAuth | 400 invalide provider/account/alias | -| `PUT /api/oauth/accounts/pause` | Suspendre/reprendre Anthropic ou un compte OAuth générique. Body `{ provider, accountId, paused }` ; la suspension du compte actif sélectionne un autre compte utilisable s’il existe. | 400 fournisseur non pris en charge ou body invalide ; 404 compte absent ; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | Suspendre/reprendre Anthropic ou un compte OAuth générique. Body `{ provider, accountId, paused }` ; la suspension du compte actif sélectionne un autre compte utilisable s’il existe. | 400 fournisseur non pris en charge ou body invalide ; 404 compte absent ; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; état de suspension enregistré ; conserver le compte et cet état, ajouter `retryAffinityReset: true` et réessayer, même après un redémarrage du serveur) | | `GET, POST, DELETE /api/providers/keys` | Répertorier les clés de fournisseur masquées, en ajouter ou en activer une, ou en supprimer une | 400 saisie invalide ; 404 fournisseur ou clé manquante | | `PUT /api/providers/keys/active` | Sélectionnez la clé active d'un fournisseur | 400 saisie invalide ; 404 provider/key manquant | | `PUT /api/providers/keys/alias` | Définir ou supprimer un alias de clé de fournisseur | 400 saisie invalide ; 404 provider/key manquant | | `GET, POST, PATCH, DELETE /api/keys` | Répertorier, créer, modifier ou supprimer les clés d'admission du plan de données | 400 corps ou identifiant invalide ; 404 clé manquante | +| `PUT /api/oauth/accounts/affinity-reset` | Confirmer le reset des associations de conversations du provider sans exiger de compte. Body `{ provider }` ; succès : `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +Le tableau de bord conserve l’avertissement tant que la publication du reset n’est pas confirmée, même si le compte a disparu. Un échec de chargement, une suspension ou une réauthentification requise conserve la tentative avec un message. Une fois l’absence confirmée par la liste, Réessayer appelle `PUT /api/oauth/accounts/affinity-reset` avec `{ provider }` et attend `affinityResetConfirmed: true` avant d’effacer l’avertissement. Cette opération supprime toutes les associations, y compris celles créées depuis le premier échec, sans sélectionner ni réactiver de compte ; une liste vide est acceptée. Une suppression avec un reset en attente envoie `retryAffinityReset=true`. En cas d’échec de publication, elle renvoie 503 `affinity_reset_not_persisted`, `deletionCommitted: true` et `retryable: true`. Le compte est déjà supprimé : réessayez le reset seul, pas la suppression. Une suppression ordinaire conserve les associations des comptes restants. Ce nettoyage est limité aux instances Anthropic OAuth enregistrées et conserve les contrôles d’authentification et d’origine de gestion. Les réponses qui répertorient les identifiants sont délibérément masquées. Les jetons d'accès OAuth et les clés API complètes des fournisseurs ne sont pas renvoyés aux clients du tableau de bord. diff --git a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md index 24c16ec5c69..5ba526d7f2b 100644 --- a/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ja/reference/cli/providers-accounts.md @@ -103,7 +103,7 @@ ocx login anthropic 実行中のプロキシを介してプロバイダー アカウントと API キー プールを一覧表示し、切り替えます。出荷されたヘルプ画面は次のとおりです。 ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex account pool, OAuth accounts and API keys (identifiers shown masked as the API returns them). history openai [--limit <1-200>] Recent routing decisions for one Codex pool account. @@ -117,6 +117,7 @@ pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. clear-cooldown Drop a cooldown the proxy set after an upstream failure. +reset-affinity [--json] Confirm all session bindings are cleared, even with no accounts; selection and pause state stay unchanged. Run on the Home; a Remote Link Child names the Home and sends no request. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -131,7 +132,7 @@ main Manage the Codex App logi Codex pool selection applies to the next request after clearing existing affinity; in-flight requests keep their captured account. ``` -サブコマンドはプロキシが実行されている必要があり、記録されたランタイムポートを自動解決します。ただし `import-orca` は例外で、プレビューはローカルのみ、`import-orca --apply` はプロキシの停止が必要です。操作が成功した場合は 0 で終了します。無効な使用法、不明なプロバイダーまたはアカウント/キー ID、到達不能なプロキシ、または API エラーが発生した場合は 1 で終了します。資格情報フィールドは、管理 API が返したとおりに表示されます (マスキングを含む)。生の API キーと OAuth トークンは決して返されません。表示の利便性は、ダッシュボードと同様にクライアント側で合成されます。`main` は、`openai` アカウント プール内の Codex アプリ ログインの CLI エイリアスであり、電子メールのない OAuth アカウントは `Account N` として表示され、プラン/ラベル列はプラン、マスクされた電子メール、ラベル、およびマスクされたキーにわたってフォールバックされます。 +サブコマンドはプロキシが実行されている必要があり、記録されたランタイムポートを自動解決します。ただし `import-orca` は例外で、プレビューはローカルのみ、`import-orca --apply` はプロキシの停止が必要です。操作が成功した場合は 0 で終了します。無効な使用法、不明なプロバイダーまたはアカウント/キー ID、到達不能なプロキシ、または API エラーが発生した場合は通常 1 で終了します。ただし `reset-affinity` は HTTP 404 で終了コード 4、HTTP 409 で終了コード 5 を返し、その他の API エラーや未確認の結果では 1 を返します。資格情報フィールドは、管理 API が返したとおりに表示されます (マスキングを含む)。生の API キーと OAuth トークンは決して返されません。表示の利便性は、ダッシュボードと同様にクライアント側で合成されます。`main` は、`openai` アカウント プール内の Codex アプリ ログインの CLI エイリアスであり、電子メールのない OAuth アカウントは `Account N` として表示され、プラン/ラベル列はプラン、マスクされた電子メール、ラベル、およびマスクされたキーにわたってフォールバックされます。 `--json` アカウント行では、次の一般的な形状が使用されます (オプションのフィールドが使用できない場合は省略されます)。 diff --git a/docs-site/src/content/docs/ja/reference/management-api.md b/docs-site/src/content/docs/ja/reference/management-api.md index 86e5f8c1831..dff47bd756d 100644 --- a/docs-site/src/content/docs/ja/reference/management-api.md +++ b/docs-site/src/content/docs/ja/reference/management-api.md @@ -247,15 +247,18 @@ Aside プロファイルの変更はこの場合でも一つだけ保存しま | `GET /api/oauth/status` | 1 つのプロバイダーの OAuth フローをポーリングする | 400 不明なプロバイダー | | `POST /api/oauth/logout` |選択したプロバイダー資格情報を削除します | 400 不明なプロバイダー。 `oauth_mutation_busy` | | `GET, DELETE /api/oauth/accounts` | マスクされたアカウントを一覧表示するか、アカウントを 1 つ削除する Kiro の行には自動選択の `autoSelectable` と、除外時には閉じた集合の `skipReason` が含まれます。アクティブな単一アカウントは送信を続けられ、クォータ取得は任意です。 | 400 無効なプロバイダー/ID。 404 アカウントがありません。 `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` |アクティブな OAuth アカウントを選択します | 400 無効なプロバイダー/アカウント。 `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` |アクティブな OAuth アカウントを選択します | 400 無効なプロバイダー/アカウント。 `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; アカウント選択は保存済み。すでに有効でも同じリクエストを再送信) | | `GET, PUT, PATCH /api/oauth/accounts/pool` | Anthropic OAuth プール ポリシーの読み取りまたは更新 | 400 非 Anthropic プロバイダーまたは無効なポリシー | | `POST /api/oauth/accounts/clear-cooldown` | 1 つの OAuth アカウントのランタイム クールダウンをクリアする | 400 無効なプロバイダー/アカウント | | `PUT /api/oauth/accounts/alias` | OAuth アカウント エイリアスを設定またはクリアする | 400 無効なプロバイダー/アカウント/エイリアス | -| `PUT /api/oauth/accounts/pause` | Anthropic または汎用 OAuth アカウントを一時停止・再開。Body `{ provider, accountId, paused }`。アクティブなアカウントを停止すると、利用可能な別のアカウントがあれば切り替えます。 | 400 未対応のプロバイダーまたは無効な body；404 アカウントなし；`oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | Anthropic または汎用 OAuth アカウントを一時停止・再開。Body `{ provider, accountId, paused }`。アクティブなアカウントを停止すると、利用可能な別のアカウントがあれば切り替えます。 | 400 未対応のプロバイダーまたは無効な body；404 アカウントなし；`oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; 一時停止状態は保存済み。同じアカウントと停止状態を維持し、`retryAffinityReset: true` を追加して再送信。サーバー再起動後もリセットを確認) | | `GET, POST, DELETE /api/providers/keys` |マスクされたプロバイダー キーを一覧表示し、1 つを追加/アクティブ化するか、1 つを削除します。 400 無効な入力。 404 プロバイダー/キーがありません | | `PUT /api/providers/keys/active` |プロバイダーのアクティブなキーを選択します | 400 無効な入力。 404 プロバイダー/キーがありません | | `PUT /api/providers/keys/alias` |プロバイダー キー エイリアスを設定またはクリアする | 400 無効な入力。 404 プロバイダー/キーがありません | | `GET, POST, PATCH, DELETE /api/keys` |データ プレーン アドミッション キーの一覧表示、作成、編集、または削除 | 400 無効な本文/ID。 404 キーがありません | +| `PUT /api/oauth/accounts/affinity-reset` | アカウントを必要とせず provider 全体の会話接続リセットの保存を確認します。Body `{ provider }`。成功時 `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +ダッシュボードは、対象アカウントが消えてもリセットの保存が確認されるまで警告を保持します。一覧の読み込み失敗や、アクティブアカウントの一時停止・再認証時も通知と再試行を保持します。一覧で不在を確認した後の再試行は、`{ provider }` で `PUT /api/oauth/accounts/affinity-reset` を呼び、`affinityResetConfirmed: true` を受け取った場合のみ警告を消します。この操作は最初の失敗後に作られた接続を含む全接続を消しますが、アカウントの選択・再開は行わず、一覧が空でも実行できます。未確認のリセットがある状態での削除は `retryAffinityReset=true` を送り、保存失敗時は 503 `affinity_reset_not_persisted`、`deletionCommitted: true`、`retryable: true` を返します。アカウントは削除済みなので、削除ではなくリセット専用 API を再試行してください。通常の削除は残るアカウントの接続を保持します。対象は登録済み Anthropic OAuth インスタンスに限り、既存の管理認証とオリジン検査を維持します。 資格情報リストの応答は意図的にマスクされます。 OAuth アクセス トークンと完全なプロバイダー API キーはダッシュボード クライアントに返されません。 diff --git a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md index e88c51adf6d..baed5fbfe5d 100644 --- a/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ko/reference/cli/providers-accounts.md @@ -213,7 +213,7 @@ Luna 메타데이터임을 표시해 사용합니다. 목록에 보인다는 사 실행 중인 프록시를 통해 제공자 계정과 API 키 풀을 나열하고 전환합니다. 제공되는 도움말 표면은 다음과 같습니다: ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex account pool, OAuth accounts and API keys (identifiers shown masked as the API returns them). history openai [--limit <1-200>] Recent routing decisions for one Codex pool account. @@ -227,6 +227,7 @@ pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. clear-cooldown Drop a cooldown the proxy set after an upstream failure. +reset-affinity [--json] Confirm all session bindings are cleared, even with no accounts; selection and pause state stay unchanged. Run on the Home; a Remote Link Child names the Home and sends no request. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -241,7 +242,7 @@ main Manage the Codex App logi Codex pool selection applies to the next request after clearing existing affinity; in-flight requests keep their captured account. ``` -하위 명령은 프록시가 실행 중이어야 하며 기록된 런타임 포트를 자동으로 찾습니다. 다만 `import-orca`는 예외로, 미리보기는 로컬 전용이고 `import-orca --apply`는 프록시가 멈춰 있어야 합니다. 성공한 작업은 종료 코드 0으로 끝납니다. 잘못된 사용, 알 수 없는 제공자 또는 계정/키 id, 도달할 수 없는 프록시, API 실패는 종료 코드 1로 끝납니다. 자격 증명 필드는 관리 API가 반환한 그대로 표시됩니다(마스킹도 그대로 포함됩니다). 원시 API 키와 OAuth 토큰은 절대 반환하지 않습니다. 표시 편의 기능은 대시보드와 마찬가지로 클라이언트 쪽에서 합성합니다. `main`은 `openai` 계정 풀의 Codex App 로그인에 대한 CLI 별칭이고, 이메일이 없는 OAuth 계정은 `Account N`으로 표시되며, plan/label 열은 plan, 마스킹된 이메일, label, 마스킹된 키 순으로 대체합니다. +하위 명령은 프록시가 실행 중이어야 하며 기록된 런타임 포트를 자동으로 찾습니다. 다만 `import-orca`는 예외로, 미리보기는 로컬 전용이고 `import-orca --apply`는 프록시가 멈춰 있어야 합니다. 성공한 작업은 종료 코드 0으로 끝납니다. 잘못된 사용, 알 수 없는 제공자 또는 계정/키 id, 도달할 수 없는 프록시, API 실패는 기본적으로 종료 코드 1로 끝납니다. 단, `reset-affinity`는 HTTP 404에 종료 코드 4, HTTP 409에 종료 코드 5를 반환하며, 그 밖의 API 실패나 확인되지 않은 결과는 종료 코드 1을 반환합니다. 자격 증명 필드는 관리 API가 반환한 그대로 표시됩니다(마스킹도 그대로 포함됩니다). 원시 API 키와 OAuth 토큰은 절대 반환하지 않습니다. 표시 편의 기능은 대시보드와 마찬가지로 클라이언트 쪽에서 합성합니다. `main`은 `openai` 계정 풀의 Codex App 로그인에 대한 CLI 별칭이고, 이메일이 없는 OAuth 계정은 `Account N`으로 표시되며, plan/label 열은 plan, 마스킹된 이메일, label, 마스킹된 키 순으로 대체합니다. `--json` 계정 행은 다음 공통 형태를 사용합니다(사용할 수 없는 필드는 생략됩니다): diff --git a/docs-site/src/content/docs/ko/reference/management-api.md b/docs-site/src/content/docs/ko/reference/management-api.md index ae9420f8118..af4e90631d3 100644 --- a/docs-site/src/content/docs/ko/reference/management-api.md +++ b/docs-site/src/content/docs/ko/reference/management-api.md @@ -255,16 +255,19 @@ Aside 프로필 변경은 이때도 한 가지를 저장합니다. 확인을 보 | `GET /api/oauth/status` | 하나의 provider OAuth 흐름을 조회합니다 | 400 알 수 없는 provider | | `POST /api/oauth/logout` | 선택된 provider 자격 증명을 제거합니다 | 400 알 수 없는 provider; `oauth_mutation_busy` | | `GET, DELETE /api/oauth/accounts` | 마스킹된 계정을 나열하거나 계정 하나를 제거합니다 Kiro 행에는 자동 선택 가능 여부인 `autoSelectable`과 제외 시 닫힌 집합의 `skipReason`이 포함됩니다. 활성 단일 계정은 여전히 요청을 보낼 수 있고 quota 조회는 선택 사항입니다. | 400 잘못된 provider/id; 404 계정 없음; `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` | 활성 OAuth 계정을 선택합니다 | 400 잘못된 provider/account; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` | 활성 OAuth 계정을 선택합니다 | 400 잘못된 provider/account; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; 계정 선택은 저장됨; 이미 활성 상태여도 동일 요청을 재전송) | | `GET, PUT, PATCH /api/pool/settings` | 모든 pool 종류(codex, anthropic, generic)의 policy를 읽거나 업데이트합니다. 세 종류 모두 같은 키로 응답하고, 해당 종류가 실제로 적용하는 필드는 `supported`에 나옵니다 | 400 알 수 없는 provider, 해당 종류가 지원하지 않는 필드, 잘못된 값 | | `GET, PUT, PATCH /api/oauth/accounts/pool` | Anthropic과 일반 OAuth provider의 기존 pool policy입니다. `/api/pool/settings`로 대체되었고 기존 클라이언트를 위해 유지합니다 | 400 codex 또는 API 키 provider, 잘못된 policy | | `POST /api/oauth/accounts/clear-cooldown` | OAuth 계정 하나의 런타임 cooldown을 지웁니다 | 400 잘못된 provider/account | | `PUT /api/oauth/accounts/alias` | OAuth 계정 alias를 설정하거나 지웁니다 | 400 잘못된 provider/account/alias | -| `PUT /api/oauth/accounts/pause` | Anthropic 또는 일반 OAuth 계정을 정지·재개합니다. Body `{ provider, accountId, paused }`. 활성 계정을 정지하면 사용 가능한 다른 계정이 있을 때 전환합니다. | 400 지원하지 않는 provider 또는 잘못된 body; 404 계정 없음; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | Anthropic 또는 일반 OAuth 계정을 정지·재개합니다. Body `{ provider, accountId, paused }`. 활성 계정을 정지하면 사용 가능한 다른 계정이 있을 때 전환합니다. | 400 지원하지 않는 provider 또는 잘못된 body; 404 계정 없음; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; 일시정지 상태는 저장됨; 계정과 정지 상태를 유지하고 `retryAffinityReset: true`를 추가해 재시도하면 서버 재시작 후에도 초기화를 확인) | | `GET, POST, DELETE /api/providers/keys` | 마스킹된 provider key를 나열, 추가/활성화, 또는 제거합니다 | 400 잘못된 입력; 404 provider/key 없음 | | `PUT /api/providers/keys/active` | provider의 활성 key를 선택합니다 | 400 잘못된 입력; 404 provider/key 없음 | | `PUT /api/providers/keys/alias` | provider-key alias를 설정하거나 지웁니다 | 400 잘못된 입력; 404 provider/key 없음 | | `GET, POST, PATCH, DELETE /api/keys` | 데이터 평면 admission key를 나열, 생성, 수정, 또는 삭제합니다 | 400 잘못된 본문/id; 404 key 없음 | +| `PUT /api/oauth/accounts/affinity-reset` | 계정 없이 provider 전체의 대화 연결 초기화 저장을 확인합니다. Body `{ provider }`; 성공 시 `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +대시보드는 저장된 계정이 사라져도 초기화 저장이 확인되기 전에는 경고를 보존합니다. 목록 로드 실패나 활성 계정의 일시정지·재인증 상태에서도 안내와 재시도를 유지합니다. 로드된 목록에서 계정 부재가 확인되면 재시도는 `{ provider }`로 `PUT /api/oauth/accounts/affinity-reset`을 호출하고, `affinityResetConfirmed: true`를 받아야 경고를 지웁니다. 이 작업은 최초 실패 이후 생성된 연결까지 모두 지우지만 계정을 선택·재개하지 않으며 빈 목록에서도 가능합니다. 초기화가 남은 상태의 계정 삭제는 `retryAffinityReset=true`를 보내고, 게시 실패 시 503 `affinity_reset_not_persisted`, `deletionCommitted: true`, `retryable: true`를 반환합니다. 계정은 이미 삭제됐으므로 삭제 대신 초기화 전용 API를 재시도하세요. 일반 삭제는 남은 계정의 연결을 보존합니다. 초기화 전용 정리는 등록된 Anthropic OAuth 인스턴스만 지원하며 기존 관리 인증·출처 검사를 유지합니다. 자격 증명 목록 응답은 의도적으로 마스킹됩니다. OAuth access token과 완전한 provider API key는 대시보드 클라이언트에 반환되지 않습니다. diff --git a/docs-site/src/content/docs/reference/cli/providers-accounts.md b/docs-site/src/content/docs/reference/cli/providers-accounts.md index edba232756a..4f854acd9d2 100644 --- a/docs-site/src/content/docs/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/reference/cli/providers-accounts.md @@ -589,7 +589,7 @@ List and switch provider accounts and API-key pools through the running proxy. T surface is: ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex account pool, OAuth accounts and API keys (identifiers shown masked as the API returns them). history openai [--limit <1-200>] Recent routing decisions for one Codex pool account. @@ -603,6 +603,7 @@ pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. clear-cooldown Drop a cooldown the proxy set after an upstream failure. +reset-affinity [--json] Confirm all session bindings are cleared, even with no accounts; selection and pause state stay unchanged. Run on the Home; a Remote Link Child names the Home and sends no request. strategy [] Pool placement strategy; least-loaded is Kiro-only. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -621,7 +622,7 @@ A selection-order change applies from the next unbound request and never moves a Account subcommands require the proxy to be running and auto-resolve its recorded runtime port, except `import-orca`: preview is local-only, and `import-orca --apply` requires the proxy to be stopped. Successful operations exit 0. Invalid usage, an unknown provider or account/key id, an unreachable -proxy, or an API failure exits 1. Credential fields are displayed exactly as the management API +proxy, or an API failure normally exits 1. For `reset-affinity`, HTTP 404 exits 4 and HTTP 409 exits 5; other API failures and unconfirmed results exit 1. Credential fields are displayed exactly as the management API returns them (including its masking); raw API keys and OAuth tokens are never returned. Display conveniences are synthesized client-side, same as the dashboard: `main` is the CLI alias for the Codex App login in the `openai` account pool, OAuth accounts without an email appear as `Account N`, diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 0bd240258a0..c8dd980ae47 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -654,8 +654,8 @@ Labels are read on login/import or refresh, never inferred from quota percentage | `POST /api/oauth/logout` | Remove the selected provider credential | 400 unknown provider; `oauth_mutation_busy` | | `GET /api/oauth/accounts` | List masked accounts; Anthropic and generic OAuth account rows include their `paused` state. Kiro rows include `autoSelectable` and a closed `skipReason` when excluded from automatic selection; an active singleton may still send. Quota remains opt-in. | 400 invalid provider | | `DELETE /api/oauth/accounts` | Remove one account | 400 invalid provider/id; 404 account missing; `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` | Select the active OAuth account | 400 invalid provider/account; 404 account missing; 409 account paused; `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/pause` | Pause or resume one Anthropic or generic OAuth account. Body `{ provider, accountId, paused }`; pausing the active account selects the next usable account when available. Pause is durable and independent of pool enablement; resume preserves health and credentials. | 400 unsupported provider or invalid body; 404 account missing; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` | Select the active OAuth account | 400 invalid provider/account; 404 account missing; 409 account paused; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; account change saved; resend the same request even if already active) | +| `PUT /api/oauth/accounts/pause` | Pause or resume one Anthropic or generic OAuth account. Body `{ provider, accountId, paused }`; pausing the active account selects the next usable account when available. Pause is durable and independent of pool enablement; resume preserves health and credentials. | 400 unsupported provider or invalid body; 404 account missing; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; pause state saved; preserve the account and pause state, add `retryAffinityReset: true`, and retry to confirm the reset even after a server restart) | | `GET, PUT, PATCH /api/pool/settings` | Read or update pool policy for any kind (codex, anthropic, generic); answers with the same keys for all three and declares in `supported` which the kind honours | 400 unknown provider, a field the kind does not support, or an invalid value | | `GET, PUT, PATCH /api/oauth/accounts/pool` | Legacy per-pool policy for Anthropic and generic OAuth providers; superseded by `/api/pool/settings` and kept for existing clients | 400 codex or api-key provider, or invalid policy | | `POST /api/oauth/accounts/clear-cooldown` | Clear one OAuth account's runtime cooldown | 400 invalid provider/account | @@ -664,6 +664,9 @@ Labels are read on login/import or refresh, never inferred from quota percentage | `PUT /api/providers/keys/active` | Select a provider's active key | 400 invalid input; 404 provider/key missing | | `PUT /api/providers/keys/alias` | Set or clear a provider-key alias | 400 invalid input; 404 provider/key missing | | `GET, POST, PATCH, DELETE /api/keys` | List, create, edit, or delete data-plane admission keys | 400 invalid body/id; 404 key missing | +| `PUT /api/oauth/accounts/affinity-reset` | Confirm a provider-wide session-affinity reset without requiring an account. Body `{ provider }`; success includes `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +The dashboard retains an unconfirmed affinity-reset warning even when the saved account disappears. Loading errors and paused or reauthentication-blocked active targets keep their retry with feedback. Once a loaded list confirms absence, Retry uses `PUT /api/oauth/accounts/affinity-reset` with `{ provider }`; it clears the warning only after `affinityResetConfirmed: true`. This provider-wide operation clears all bindings, including those created since the first failure, without selecting or resuming an account, and also works with an empty roster. Deleting an account with an outstanding reset sends `retryAffinityReset=true`; a failed publication returns 503 `affinity_reset_not_persisted`, `deletionCommitted: true` and `retryable: true`. The account is already deleted: retry the reset-only endpoint, not deletion. Plain deletion preserves surviving-account affinity. Reset-only cleanup is limited to the registered Anthropic OAuth instances and retains the ordinary management authentication/origin checks. `POST /api/oauth/login/cancel` can also dismiss the selected settled failed login attempt. It leaves stored credentials and successful login results intact; status then reports the remaining flow or idle. Cancelling an in-progress attempt still records its cancelled outcome. diff --git a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md index 584eacdb910..8aeaa9cccd8 100644 --- a/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/ru/reference/cli/providers-accounts.md @@ -118,7 +118,7 @@ ocx login anthropic Поставляемая help-surface выглядит так: ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex account pool, OAuth accounts and API keys (identifiers shown masked as the API returns them). history openai [--limit <1-200>] Recent routing decisions for one Codex pool account. @@ -132,6 +132,7 @@ pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. clear-cooldown Drop a cooldown the proxy set after an upstream failure. +reset-affinity [--json] Confirm all session bindings are cleared, even with no accounts; selection and pause state stay unchanged. Run on the Home; a Remote Link Child names the Home and sends no request. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -148,7 +149,7 @@ Codex pool selection applies to the next request after clearing existing affinit Подкоманды требуют запущенного прокси и сами определяют записанный runtime-port, кроме `import-orca`: предпросмотр работает только локально, а `import-orca --apply` требует остановленного прокси. Успешные операции завершаются с кодом 0. Некорректное использование, неизвестный провайдер, account/key id, -недостижимый прокси или ошибка API приводят к коду 1. Поля credential'ов выводятся ровно в том +недостижимый прокси или ошибка API обычно приводят к коду 1. Для `reset-affinity` HTTP 404 означает код выхода 4, HTTP 409 — код 5; остальные ошибки API и неподтверждённые результаты дают код 1. Поля credential'ов выводятся ровно в том виде, как их возвращает management API (включая его masking); сырые API-key'и и OAuth-token'ы никогда не возвращаются. Display convenience синтезируется на стороне клиента, как и в дашборде: `main` — это alias CLI для логина Codex App внутри пула `openai`, OAuth-аккаунты без email diff --git a/docs-site/src/content/docs/ru/reference/management-api.md b/docs-site/src/content/docs/ru/reference/management-api.md index 3d59f47db89..2006ad30eb1 100644 --- a/docs-site/src/content/docs/ru/reference/management-api.md +++ b/docs-site/src/content/docs/ru/reference/management-api.md @@ -271,16 +271,19 @@ Endpoint'ы storage cleanup могут перемещать или навсег | `GET /api/oauth/status` | Опрашивать OAuth-flow одного провайдера | 400 unknown provider | | `POST /api/oauth/logout` | Удалить сохранённый credential выбранного провайдера | 400 unknown provider; `oauth_mutation_busy` | | `GET, DELETE /api/oauth/accounts` | Показать список masked-аккаунтов или удалить один аккаунт Строки Kiro содержат `autoSelectable` и закрытый `skipReason` при исключении из автоматического выбора; единственный активный аккаунт всё ещё может отправлять запросы. Чтение квоты остаётся необязательным. | 400 invalid provider/id; 404 account missing; `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` | Выбрать активный OAuth-аккаунт | 400 invalid provider/account; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` | Выбрать активный OAuth-аккаунт | 400 invalid provider/account; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; выбор сохранён; повторите тот же запрос, даже если аккаунт уже активен) | | `GET, PUT, PATCH /api/pool/settings` | Прочитать или обновить policy пула любого вида (codex, anthropic, generic); все три отвечают одинаковыми ключами, а поля, которые вид действительно применяет, перечислены в `supported` | 400 неизвестный provider, поле, которое вид не поддерживает, или недопустимое значение | | `GET, PUT, PATCH /api/oauth/accounts/pool` | Прежняя policy пула для Anthropic и обычных OAuth-провайдеров; заменена на `/api/pool/settings` и сохранена для существующих клиентов | 400 codex или api-key provider, либо недопустимая policy | | `POST /api/oauth/accounts/clear-cooldown` | Очистить runtime cooldown одного OAuth-аккаунта | 400 invalid provider/account | | `PUT /api/oauth/accounts/alias` | Задать или очистить alias OAuth-аккаунта | 400 invalid provider/account/alias | -| `PUT /api/oauth/accounts/pause` | Приостановить/возобновить Anthropic или обычный OAuth-аккаунт. Body `{ provider, accountId, paused }`; при паузе активного аккаунта выбирается другой пригодный аккаунт, если он есть. | 400 неподдерживаемый provider или неверный body; 404 аккаунт не найден; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | Приостановить/возобновить Anthropic или обычный OAuth-аккаунт. Body `{ provider, accountId, paused }`; при паузе активного аккаунта выбирается другой пригодный аккаунт, если он есть. | 400 неподдерживаемый provider или неверный body; 404 аккаунт не найден; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; состояние паузы сохранено; сохраните тот же аккаунт и состояние, добавьте `retryAffinityReset: true` и повторите запрос, в том числе после перезапуска сервера) | | `GET, POST, DELETE /api/providers/keys` | Показать список masked provider-key'ов, добавить/активировать один или удалить один | 400 invalid input; 404 provider/key missing | | `PUT /api/providers/keys/active` | Выбрать активный ключ провайдера | 400 invalid input; 404 provider/key missing | | `PUT /api/providers/keys/alias` | Задать или очистить alias provider-key'а | 400 invalid input; 404 provider/key missing | | `GET, POST, PATCH, DELETE /api/keys` | Показать список, создать, отредактировать или удалить admission key data plane | 400 invalid body/id; 404 key missing | +| `PUT /api/oauth/accounts/affinity-reset` | Подтвердить запись сброса привязок разговоров provider без обязательного аккаунта. Body `{ provider }`; успех: `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +Панель сохраняет предупреждение до подтверждения записи сброса, даже если аккаунт исчез. Ошибка загрузки, пауза или необходимость повторного входа не удаляют попытку. После подтверждения отсутствия аккаунта повтор вызывает `PUT /api/oauth/accounts/affinity-reset` с `{ provider }` и снимает предупреждение только при `affinityResetConfirmed: true`. Операция очищает все привязки, включая созданные после первого сбоя, не выбирает и не возобновляет аккаунты и работает при пустом списке. Удаление с незавершённым сбросом передаёт `retryAffinityReset=true`; ошибка записи возвращает 503 `affinity_reset_not_persisted`, `deletionCommitted: true` и `retryable: true`. Аккаунт уже удалён: повторяйте только сброс. Обычное удаление сохраняет привязки остальных аккаунтов. Сброс доступен только зарегистрированным Anthropic OAuth instances и сохраняет проверки управляющей аутентификации и источника. Ответы со списками credential'ов намеренно маскируются. OAuth access-token'ы и полные API-key'и провайдеров клиентам дашборда не возвращаются. diff --git a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md index f0cec9d17ac..1b91857e4ea 100644 --- a/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/tr/reference/cli/providers-accounts.md @@ -137,7 +137,7 @@ Bir sağlayıcı için saklanan OAuth kimlik bilgisini kaldırın. listeleyin ve değiştirin. Sağlanan yardım arayüzü şöyledir: ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex account pool, OAuth accounts and API keys (identifiers shown masked as the API returns them). history openai [--limit <1-200>] Recent routing decisions for one Codex pool account. @@ -151,6 +151,7 @@ pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. clear-cooldown Drop a cooldown the proxy set after an upstream failure. +reset-affinity [--json] Confirm all session bindings are cleared, even with no accounts; selection and pause state stay unchanged. Run on the Home; a Remote Link Child names the Home and sends no request. strategy [] Havuz stratejisi; least-loaded yalnızca Kiro içindir. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -168,7 +169,7 @@ A selection-order change applies from the next unbound request and never moves a `import-orca` dışındaki alt komutlar proxy'nin çalışmasını gerektirir ve kaydedilen çalışma zamanı portunu otomatik çözer; `import-orca` önizlemesi tamamen yereldir ve `import-orca --apply` proxy'nin durdurulmuş olmasını gerektirir. Başarılı işlemler 0 ile çıkar. Geçersiz kullanım, bilinmeyen bir sağlayıcı veya hesap/anahtar kimliği, erişilemeyen bir proxy veya -bir API hatası 1 ile çıkar. Kimlik bilgisi alanları tam olarak yönetim API'sinin +bir API hatası normalde 1 ile çıkar. `reset-affinity` için HTTP 404 çıkış kodu 4, HTTP 409 çıkış kodu 5 döndürür; diğer API hataları ve doğrulanmamış sonuçlar 1 döndürür. Kimlik bilgisi alanları tam olarak yönetim API'sinin döndürdüğü gibi (maskelemesi dahil) görüntülenir; ham API anahtarları ve OAuth belirteçleri asla döndürülmez. Görüntüleme kolaylıkları kontrol paneli gibi istemci tarafında sentezlenir: `main`, `openai` hesap havuzundaki Codex App diff --git a/docs-site/src/content/docs/tr/reference/management-api.md b/docs-site/src/content/docs/tr/reference/management-api.md index 33824ff6c6c..65d1b758543 100644 --- a/docs-site/src/content/docs/tr/reference/management-api.md +++ b/docs-site/src/content/docs/tr/reference/management-api.md @@ -296,15 +296,18 @@ Güvenilir ilk model listesi hazır olana kadar `/api/selected-models` ve `/api/ | `GET /api/oauth/status` | Bir sağlayıcının OAuth akışını yoklayın | 400 bilinmeyen sağlayıcı | | `POST /api/oauth/logout` | Seçilen sağlayıcı kimlik bilgisini kaldırın | 400 bilinmeyen sağlayıcı; `oauth_mutation_busy` | | `GET, DELETE /api/oauth/accounts` | Maskelenmiş hesapları listeleyin veya bir hesabı kaldırın Kiro satırları otomatik seçimden dışlandığında `autoSelectable` ve kapalı bir `skipReason` taşır; tek etkin hesap yine istek gönderebilir. Kota isteğe bağlıdır. | 400 geçersiz sağlayıcı/kimlik; 404 hesap eksik; `oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` | Aktif OAuth hesabını seçin | 400 geçersiz sağlayıcı/hesap; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` | Aktif OAuth hesabını seçin | 400 geçersiz sağlayıcı/hesap; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; hesap seçimi kaydedildi; hesap etkin olsa bile aynı isteği yeniden gönderin) | | `GET, PUT, PATCH /api/oauth/accounts/pool` | Anthropic OAuth havuz politikasını okuyun veya güncelleyin | 400 Anthropic olmayan sağlayıcı veya geçersiz politika | | `POST /api/oauth/accounts/clear-cooldown` | Bir OAuth hesabının çalışma zamanı soğuma süresini temizleyin | 400 geçersiz sağlayıcı/hesap | | `PUT /api/oauth/accounts/alias` | Bir OAuth hesap takma adını ayarlayın veya temizleyin | 400 geçersiz sağlayıcı/hesap/takma ad | -| `PUT /api/oauth/accounts/pause` | Anthropic veya genel OAuth hesabını duraklatın/sürdürün. Body `{ provider, accountId, paused }`; etkin hesap duraklatıldığında varsa kullanılabilir başka hesaba geçilir. | 400 desteklenmeyen sağlayıcı veya geçersiz body; 404 hesap yok; `oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | Anthropic veya genel OAuth hesabını duraklatın/sürdürün. Body `{ provider, accountId, paused }`; etkin hesap duraklatıldığında varsa kullanılabilir başka hesaba geçilir. | 400 desteklenmeyen sağlayıcı veya geçersiz body; 404 hesap yok; `oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; duraklatma durumu kaydedildi; hesabı ve durumu koruyup `retryAffinityReset: true` ekleyerek yeniden deneyin; sunucu yeniden başlatıldıktan sonra da sıfırlama doğrulanır) | | `GET, POST, DELETE /api/providers/keys` | Maskelenmiş sağlayıcı anahtarlarını listeleyin, bir tane ekleyin/etkinleştirin veya kaldırın | 400 geçersiz girdi; 404 sağlayıcı/anahtar eksik | | `PUT /api/providers/keys/active` | Bir sağlayıcının etkin anahtarını seçin | 400 geçersiz girdi; 404 sağlayıcı/anahtar eksik | | `PUT /api/providers/keys/alias` | Bir sağlayıcı anahtarı takma adını ayarlayın veya temizleyin | 400 geçersiz girdi; 404 sağlayıcı/anahtar eksik | | `GET, POST, PATCH, DELETE /api/keys` | Veri düzlemi kabul anahtarlarını listeleyin, oluşturun, düzenleyin veya silin | 400 geçersiz gövde/kimlik; 404 anahtar eksik | +| `PUT /api/oauth/accounts/affinity-reset` | Hesap gerektirmeden provider konuşma bağlantılarının sıfırlanmasını doğrulayın. Body `{ provider }`; başarı: `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +Pano, hesap kaybolsa bile sıfırlamanın kaydedildiği doğrulanana kadar uyarıyı korur. Yükleme hatası, duraklatma veya yeniden giriş gereksinimi denemeyi silmez. Hesabın yokluğu listeden doğrulandığında Yeniden Dene, `{ provider }` ile `PUT /api/oauth/accounts/affinity-reset` çağırır; uyarı yalnızca `affinityResetConfirmed: true` ile kaldırılır. Bu işlem ilk hatadan sonra oluşanlar dahil tüm bağlantıları temizler, hesap seçmez veya devam ettirmez ve boş listede de çalışır. Bekleyen sıfırlama varken silme `retryAffinityReset=true` gönderir; kayıt hatası 503 `affinity_reset_not_persisted`, `deletionCommitted: true` ve `retryable: true` döndürür. Hesap zaten silinmiştir: silmeyi değil yalnızca sıfırlamayı yeniden deneyin. Normal silme kalan hesapların bağlantılarını korur. İşlem kayıtlı Anthropic OAuth örnekleriyle sınırlıdır ve mevcut yönetim kimlik doğrulaması ile kaynak kontrollerini korur. Kimlik bilgisi listesi yanıtları kasıtlı olarak maskelenir. OAuth erişim belirteçleri ve eksiksiz sağlayıcı API anahtarları kontrol paneli istemcilerine diff --git a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md index cf8e96e1320..63ddac2fa64 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli/providers-accounts.md @@ -115,7 +115,7 @@ ocx login anthropic 通过正在运行的代理列出并切换提供方账号和 API 密钥池。随附的帮助输出如下： ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex account pool, OAuth accounts and API keys (identifiers shown masked as the API returns them). history openai [--limit <1-200>] Recent routing decisions for one Codex pool account. @@ -129,6 +129,7 @@ pause Hold an account out of automatic selection. resume Return a paused account to automatic selection. pause-exhausted Pause every account whose quota is spent. clear-cooldown Drop a cooldown the proxy set after an upstream failure. +reset-affinity [--json] Confirm all session bindings are cleared, even with no accounts; selection and pause state stay unchanged. Run on the Home; a Remote Link Child names the Home and sends no request. strategy [] Pool placement strategy; omit the value to read it. sticky [<1-100>] Requests a bound thread keeps on one account; omit the value to read it. priority [first|earlier|normal|later|last|-100..100|reset] Selection order; omit the value to read it. @@ -145,7 +146,7 @@ Codex pool selection applies to the next request after clearing existing affinit 除 `import-orca` 外，子命令都要求代理正在运行，CLI 会自动解析其记录的运行时端口；`import-orca` 的预览完全在本地进行，而 `import-orca --apply` 要求代理已停止。成功操作的 退出码为 0。无效用法、未知的提供方或账号/密钥 id、无法访问的代理，或 API 失败 -都会以 1 退出。凭据字段会严格按管理 API 返回的样子显示（包括其屏蔽格式）； +通常以 1 退出。但 `reset-affinity` 遇到 HTTP 404 时退出码为 4，遇到 HTTP 409 时为 5；其他 API 失败或未确认的结果退出码为 1。凭据字段会严格按管理 API 返回的样子显示（包括其屏蔽格式）； 原始 API 密钥和 OAuth token 永远不会返回。显示上的便利字段都像仪表盘一样在 客户端侧合成：`main` 是 `openai` 账号池中 Codex App 登录的 CLI 别名，没有邮箱的 OAuth 账号会显示为 `Account N`，而 plan/label 列会在 plan、屏蔽后的邮箱、label 和 diff --git a/docs-site/src/content/docs/zh-cn/reference/management-api.md b/docs-site/src/content/docs/zh-cn/reference/management-api.md index 3679c8c5390..69bb3fe27f8 100644 --- a/docs-site/src/content/docs/zh-cn/reference/management-api.md +++ b/docs-site/src/content/docs/zh-cn/reference/management-api.md @@ -237,15 +237,18 @@ Aside 配置档的变更在这种情况下仍会保存一件事：确认之后 | `GET /api/oauth/status` | 轮询某个 provider 的 OAuth 流程 | 400 provider 未知 | | `POST /api/oauth/logout` | 移除选定的 provider 凭证 | 400 provider 未知；`oauth_mutation_busy` | | `GET, DELETE /api/oauth/accounts` | 列出已脱敏账户或移除一个账户 Kiro 行包含自动选择状态 `autoSelectable`，被排除时还包含封闭集合的 `skipReason`。唯一的活动账户仍可发送请求，配额查询仍为可选。 | 400 provider/id 无效；404 账户缺失；`oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` | 选择当前活跃的 OAuth 账户 | 400 provider/账户无效；`oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` | 选择当前活跃的 OAuth 账户 | 400 provider/账户无效；`oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; 账户选择已保存；即使已激活，也请重新发送相同请求) | | `GET, PUT, PATCH /api/oauth/accounts/pool` | 读取或更新 Anthropic OAuth 池策略 | 400 非 Anthropic provider 或策略无效 | | `POST /api/oauth/accounts/clear-cooldown` | 清除一个 OAuth 账户的运行时冷却 | 400 provider/账户无效 | | `PUT /api/oauth/accounts/alias` | 设置或清除 OAuth 账户别名 | 400 provider/账户/别名无效 | -| `PUT /api/oauth/accounts/pause` | 暂停或恢复 Anthropic 或通用 OAuth 账户。Body `{ provider, accountId, paused }`；暂停活跃账户时，如有其他可用账户则切换过去。 | 400 不支持的 provider 或无效 body；404 账户不存在；`oauth_mutation_busy` | +| `PUT /api/oauth/accounts/pause` | 暂停或恢复 Anthropic 或通用 OAuth 账户。Body `{ provider, accountId, paused }`；暂停活跃账户时，如有其他可用账户则切换过去。 | 400 不支持的 provider 或无效 body；404 账户不存在；`oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; 暂停状态已保存；保留账户和暂停状态，添加 `retryAffinityReset: true` 后重试，服务器重启后也会确认重置) | | `GET, POST, DELETE /api/providers/keys` | 列出已脱敏的 provider 密钥，添加/激活一个，或移除一个 | 400 输入无效；404 provider/密钥缺失 | | `PUT /api/providers/keys/active` | 选择某个 provider 的活跃密钥 | 400 输入无效；404 provider/密钥缺失 | | `PUT /api/providers/keys/alias` | 设置或清除 provider 密钥别名 | 400 输入无效；404 provider/密钥缺失 | | `GET, POST, PATCH, DELETE /api/keys` | 列出、创建、编辑或删除数据平面准入密钥 | 400 请求体/id 无效；404 密钥缺失 | +| `PUT /api/oauth/accounts/affinity-reset` | 无需账户即可确认 provider 全部会话绑定的重置已保存。Body `{ provider }`；成功返回 `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +即使保存的账户已消失，仪表板也会保留未确认的重置警告。列表加载失败、账户暂停或需要重新认证时，会保留重试并提示。列表确认账户不存在后，重试使用 `{ provider }` 调用 `PUT /api/oauth/accounts/affinity-reset`，只有收到 `affinityResetConfirmed: true` 才清除警告。该操作会清除所有绑定，包括首次失败后创建的绑定，但不会选择或恢复账户，账户列表为空时也可执行。存在待完成重置时，删除请求发送 `retryAffinityReset=true`；保存失败返回 503 `affinity_reset_not_persisted`、`deletionCommitted: true` 和 `retryable: true`。账户已经删除，请重试专用重置接口而非删除。普通删除保留其他账户的绑定。专用清理仅适用于注册的 Anthropic OAuth 实例，并保留原有管理认证和来源检查。 凭证列表响应会刻意脱敏。OAuth 访问令牌和完整的 provider API 密钥不会返回给仪表板客户端。 diff --git a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md index 0ec96d57678..1f308227b34 100644 --- a/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md +++ b/docs-site/src/content/docs/zh-tw/reference/cli/providers-accounts.md @@ -86,7 +86,7 @@ ocx login anthropic 透過執行中的代理列出並切換供應商帳號與 API-key 池。隨附的說明介面如下： ```text -Usage: ocx account ... +Usage: ocx account ... list [provider] Codex 帳號池、OAuth 帳號與 API 金鑰（識別碼依 API 回傳遮罩顯示）。 history openai [--limit <1-200>] 單一 Codex 帳號池帳號的近期路由決策。 @@ -101,6 +101,7 @@ pause 將帳號移出自動選擇。 resume 將暫停的帳號放回自動選擇。 pause-exhausted 暫停所有配額已用盡的帳號。 clear-cooldown 清除上游失敗後設定的冷卻。 +reset-affinity [--json] 確認所有工作階段綁定均已清除，即使沒有帳號也適用；帳號選擇與暫停狀態保持不變。請在 Home 執行；Remote Link Child 只會指出 Home，不會傳送請求。 strategy [] 帳號池放置策略；省略取值即讀取目前值。 sticky [<1-100>] 已綁定執行緒在同一帳號上保留的請求數；省略取值即讀取目前值。 remove --yes 在存在檢查後移除已儲存的帳號或金鑰。 @@ -114,7 +115,7 @@ main 管理帳號池稱為 'ma Codex 池選擇套用於清除既有親和性後的下一個請求；進行中的請求保留其擷取的帳號。 ``` -除 `import-orca` 外，子指令都需要代理正在執行，CLI 自動解析其記錄的 runtime 連接埠；`import-orca` 的預覽完全在本機進行，而 `import-orca --apply` 需要代理已停止。成功的操作離開 0。無效用法、未知供應商或帳號／金鑰 id、不可達的代理或 API 失敗則離開 1。憑證欄位完全依管理 API 回傳的方式顯示（包含其遮罩）；原始 API 金鑰與 OAuth token 永不回傳。顯示便利性在客戶端合成，與儀表板相同：`main` 是 `openai` 帳號池中 Codex App 登入的 CLI 別名，無電子郵件的 OAuth 帳號顯示為 `Account N`，而 plan／label 欄位在 plan、遮罩電子郵件、label 與遮罩金鑰之間回退。 +除 `import-orca` 外，子指令都需要代理正在執行，CLI 自動解析其記錄的 runtime 連接埠；`import-orca` 的預覽完全在本機進行，而 `import-orca --apply` 需要代理已停止。成功的操作離開 0。無效用法、未知供應商或帳號／金鑰 id、不可達的代理或 API 失敗通常以 1 結束。但 `reset-affinity` 遇到 HTTP 404 時結束碼為 4，遇到 HTTP 409 時為 5；其他 API 失敗或未確認的結果則為 1。憑證欄位完全依管理 API 回傳的方式顯示（包含其遮罩）；原始 API 金鑰與 OAuth token 永不回傳。顯示便利性在客戶端合成，與儀表板相同：`main` 是 `openai` 帳號池中 Codex App 登入的 CLI 別名，無電子郵件的 OAuth 帳號顯示為 `Account N`，而 plan／label 欄位在 plan、遮罩電子郵件、label 與遮罩金鑰之間回退。 `--json` 帳號列使用此通用結構（不可用時省略可選欄位）： diff --git a/docs-site/src/content/docs/zh-tw/reference/management-api.md b/docs-site/src/content/docs/zh-tw/reference/management-api.md index f4d13b561b4..0c878cda1c8 100644 --- a/docs-site/src/content/docs/zh-tw/reference/management-api.md +++ b/docs-site/src/content/docs/zh-tw/reference/management-api.md @@ -241,8 +241,8 @@ Aside 設定檔的變更在這種情況下仍會儲存一件事：確認之後 | `POST /api/oauth/logout` | 移除所選的供應商憑證 | 400 未知供應商；`oauth_mutation_busy` | | `GET /api/oauth/accounts` | 列出遮罩帳號；Anthropic 與通用 OAuth 帳號列也會提供 `paused` 狀態。Kiro 列包含自動選取狀態 `autoSelectable`，排除時還包含封閉集合的 `skipReason`。唯一的有效帳號仍可傳送請求，配額查詢仍為選用。 | 400 無效供應商 | | `DELETE /api/oauth/accounts` | 移除一個帳號 | 400 無效供應商/id；404 帳號缺失；`oauth_mutation_busy` | -| `PUT /api/oauth/accounts/active` | 選擇現用 OAuth 帳號 | 400 無效供應商／帳號；404 帳號缺失；409 帳號已暫停；`oauth_mutation_busy` | -| `PUT /api/oauth/accounts/pause` | 暫停或恢復一個 Anthropic 或通用 OAuth 帳號。Body `{ provider, accountId, paused }`；若暫停現用帳號，且有可用帳號，會切換至下一個。暫停會持久儲存且不受帳號池開關影響，恢復保留健康狀態與憑證。 | 400 不支援的供應商或無效 body；404 帳號缺失；`oauth_mutation_busy` | +| `PUT /api/oauth/accounts/active` | 選擇現用 OAuth 帳號 | 400 無效供應商／帳號；404 帳號缺失；409 帳號已暫停；`oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `selectionCommitted: true`, `retryable: true`; 帳戶選擇已儲存；即使已啟用，也請重新傳送相同請求) | +| `PUT /api/oauth/accounts/pause` | 暫停或恢復一個 Anthropic 或通用 OAuth 帳號。Body `{ provider, accountId, paused }`；若暫停現用帳號，且有可用帳號，會切換至下一個。暫停會持久儲存且不受帳號池開關影響，恢復保留健康狀態與憑證。 | 400 不支援的供應商或無效 body；404 帳號缺失；`oauth_mutation_busy`; 503 `affinity_reset_not_persisted` (Anthropic: `pauseCommitted: true`, `retryable: true`; 暫停狀態已儲存；保留帳號和暫停狀態，加入 `retryAffinityReset: true` 後重試，伺服器重新啟動後也會確認重設) | | `GET, PUT, PATCH /api/oauth/accounts/pool` | 讀取或更新 Anthropic OAuth 池政策 | 400 非 Anthropic 供應商或無效政策 | | `POST /api/oauth/accounts/clear-cooldown` | 清除一個 OAuth 帳號的 runtime 冷卻 | 400 無效供應商／帳號 | | `PUT /api/oauth/accounts/alias` | 設定或清除 OAuth 帳號別名 | 400 無效供應商／帳號／別名 | @@ -250,6 +250,9 @@ Aside 設定檔的變更在這種情況下仍會儲存一件事：確認之後 | `PUT /api/providers/keys/active` | 選擇供應商的現用金鑰 | 400 無效輸入；404 供應商／金鑰缺失 | | `PUT /api/providers/keys/alias` | 設定或清除供應商金鑰別名 | 400 無效輸入；404 供應商／金鑰缺失 | | `GET, POST, PATCH, DELETE /api/keys` | 列出、建立、編輯或刪除 data-plane 許可金鑰 | 400 無效 body/id；404 金鑰缺失 | +| `PUT /api/oauth/accounts/affinity-reset` | 無需帳戶即可確認 provider 全部對話繫結的重設已儲存。Body `{ provider }`；成功回傳 `affinityResetConfirmed: true`. CLI: `ocx account reset-affinity anthropic --json` (`anthropic2` supported). | 400 invalid body/provider; 503 `affinity_reset_not_persisted`, `retryable: true` | + +即使儲存的帳戶已消失，儀表板也會保留未確認的重設警告。清單載入失敗、帳戶暫停或需要重新驗證時，會保留重試並提示。清單確認帳戶不存在後，重試使用 `{ provider }` 呼叫 `PUT /api/oauth/accounts/affinity-reset`，只有收到 `affinityResetConfirmed: true` 才清除警告。此操作會清除所有繫結，包括首次失敗後建立的繫結，但不會選擇或恢復帳戶，帳戶清單為空時也可執行。存在待完成重設時，刪除請求傳送 `retryAffinityReset=true`；儲存失敗回傳 503 `affinity_reset_not_persisted`、`deletionCommitted: true` 和 `retryable: true`。帳戶已經刪除，請重試專用重設端點而非刪除。一般刪除保留其他帳戶的繫結。專用清理僅適用於註冊的 Anthropic OAuth 執行個體，並保留原有管理驗證與來源檢查。 憑證清單回應被刻意遮罩。OAuth access token 與完整的供應商 API 金鑰不回傳給儀表板客戶端。 diff --git a/gui/src/components/provider-workspace/ProviderAuthPanel.tsx b/gui/src/components/provider-workspace/ProviderAuthPanel.tsx index a76d3c60746..21da80c8bb7 100644 --- a/gui/src/components/provider-workspace/ProviderAuthPanel.tsx +++ b/gui/src/components/provider-workspace/ProviderAuthPanel.tsx @@ -536,6 +536,14 @@ export default function ProviderAuthPanel({ )} + {authHandlers.pendingAffinityResets?.[item.name] && ( +
+ {t("prov.affinityResetPending")} + {authHandlers.onRetryAffinityReset && } +
+ )} {accountLoadState === "loading" && accounts.length === 0 && (