diff --git a/cookbook/rl/graphcredit/README.md b/cookbook/rl/graphcredit/README.md
new file mode 100644
index 000000000..8922ffecf
--- /dev/null
+++ b/cookbook/rl/graphcredit/README.md
@@ -0,0 +1,20 @@
+# GraphCredit Gate 0
+
+该目录只包含离线 Gate 0 探针,不进入 Twinkle 核心训练路径。
+
+Gate 0 使用确定性状态机生成多轮轨迹,验证以下前提:
+
+- 环境显式状态键覆盖率;
+- 同一 prompt 下同状态动作的支撑度;
+- 状态图 credit 对正确动作和错误动作的排序能力;
+- 无成功终态时 graph advantage 是否严格为零;
+- 纯 CPU 图构造和 advantage 计算耗时;
+- 是否产生额外 policy forward。
+
+运行:
+
+```bash
+PYTHONPATH=src python cookbook/rl/graphcredit/gate0.py
+```
+
+Gate 0 未通过时,不进入 `TurnCredit` 或 state-graph advantage 的核心实现。
diff --git a/cookbook/rl/graphcredit/README_GATE1.md b/cookbook/rl/graphcredit/README_GATE1.md
new file mode 100644
index 000000000..0a3101def
--- /dev/null
+++ b/cookbook/rl/graphcredit/README_GATE1.md
@@ -0,0 +1,69 @@
+# GraphCredit Gate 1
+
+该实验在保持 rollout、组大小、模型、LoRA 配置、更新步数和评测任务完全一致的条件下,
+比较 episode-level GRPO 与 `StateGraphAdvantage`。
+
+## 任务
+
+二维网格只要求连续两步靠近目标。错误方向会立即终止 episode。episode reward 只使用
+最终成功或失败:
+
+- 成功:`1.0`
+- 失败:`0.0`
+
+因此 GRPO 只能把最终结果广播给 episode 内所有动作,而 GraphCredit 可以根据
+`pre_state` 分组,给“当前动作正确但后续失败”的中间动作更高信用。
+
+## A2 运行
+
+```bash
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+export PYTHONPATH="$PWD/src"
+
+python cookbook/rl/graphcredit/gate1.py \
+ --model-path /workspace/shared_assets/models/Qwen/Qwen3-0.6B \
+ --mode grpo \
+ --steps 20 \
+ --eval-tasks 24 \
+ --group-size 8 \
+ --device npu:0 \
+ --output-dir outputs/graphcredit-gate1/grpo
+
+python cookbook/rl/graphcredit/gate1.py \
+ --model-path /workspace/shared_assets/models/Qwen/Qwen3-0.6B \
+ --mode graphcredit \
+ --steps 20 \
+ --eval-tasks 24 \
+ --group-size 8 \
+ --device npu:0 \
+ --output-dir outputs/graphcredit-gate1/graphcredit
+```
+
+两次运行必须使用相同 `--seed`、模型、任务范围、group size 和训练步数。脚本输出:
+
+- `adapter/`:训练后的 LoRA;
+- `result.json`:逐步 reward、loss、耗时、峰值 HBM 和 GraphCredit diagnostics;
+- 标准输出:每一步一行 JSON,便于远端日志采集。
+
+也可以使用统一启动脚本:
+
+```bash
+bash cookbook/rl/graphcredit/run_gate1_a2.sh grpo 20 24
+bash cookbook/rl/graphcredit/run_gate1_a2.sh graphcredit 20 24
+```
+
+可通过环境变量覆盖难度和周期性评测,例如:
+
+```bash
+DISTANCE=3 GRID_SIZE=7 EVAL_EVERY=4 \
+ bash cookbook/rl/graphcredit/run_gate1_a2.sh graphcredit 12 16
+```
+
+`--steps` 表示成功优化更新次数,不是跳过组也计数的尝试次数。候选训练组最多允许
+`MAX_STEP_ATTEMPTS` 倍尝试,避免两臂实际更新预算不同。
+
+## 证据边界
+
+该实验验证 advantage consumer 的训练方向,不替代 Twinkle 完整 `GRPOLoss` 和生产训练
+入口。它没有 reference KL、多 epoch PPO 或异步 rollout。只有在 held-out success
+稳定优于同预算 GRPO 后,才进入真实 Agentic 任务和多种子验证。
diff --git a/cookbook/rl/graphcredit/gate0.py b/cookbook/rl/graphcredit/gate0.py
new file mode 100644
index 000000000..70f43424c
--- /dev/null
+++ b/cookbook/rl/graphcredit/gate0.py
@@ -0,0 +1,347 @@
+"""Offline Gate 0 probe for the proposed TurnCredit state graph.
+
+This probe deliberately does not import or modify Twinkle's rollout path. It
+tests the proposed state-key and graph semantics on deterministic trajectories
+before any core implementation is allowed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import statistics
+import time
+from collections import defaultdict, deque
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Iterable
+
+
+@dataclass(frozen=True)
+class Transition:
+ prompt_id: str
+ rollout_id: str
+ turn_id: int
+ pre_state: str
+ action: str
+ post_state: str
+ action_correct: bool
+ terminal_success: bool | None
+
+
+@dataclass(frozen=True)
+class TurnScore:
+ pre_state: str
+ action: str
+ action_correct: bool
+ graph_return: float
+ graph_advantage: float
+
+
+def _build_rollout(
+ prompt_id: str,
+ rollout_id: str,
+ choices: list[str],
+) -> list[Transition]:
+ transitions: list[Transition] = []
+ state = f'{prompt_id}:s0'
+ for turn_id, choice in enumerate(choices):
+ correct = choice == 'ok'
+ if correct:
+ if turn_id == 2:
+ post_state = f'{prompt_id}:success'
+ else:
+ post_state = f'{prompt_id}:s{turn_id + 1}'
+ else:
+ post_state = f'{prompt_id}:dead{turn_id}'
+ transitions.append(
+ Transition(
+ prompt_id=prompt_id,
+ rollout_id=rollout_id,
+ turn_id=turn_id,
+ pre_state=state,
+ action=choice,
+ post_state=post_state,
+ action_correct=correct,
+ terminal_success=True if correct and turn_id == 2 else None,
+ ))
+ if not correct:
+ break
+ state = post_state
+ return transitions
+
+
+def _build_fixture() -> list[Transition]:
+ paths = (
+ ('ok', 'ok', 'ok'),
+ ('bad',),
+ ('ok', 'bad'),
+ ('ok', 'ok', 'bad'),
+ ('ok', 'ok', 'ok'),
+ ('bad',),
+ ('ok', 'bad'),
+ ('ok', 'ok', 'bad'),
+ )
+ records: list[Transition] = []
+ for prompt_index in range(8):
+ prompt_id = f'p{prompt_index}'
+ for rollout_index, choices in enumerate(paths):
+ records.extend(_build_rollout(prompt_id, f'r{rollout_index}', list(choices)))
+ return records
+
+
+def _graph_distances(records: Iterable[Transition]) -> dict[str, int]:
+ edges: dict[str, set[str]] = defaultdict(set)
+ terminal_success: set[str] = set()
+ nodes: set[str] = set()
+ for record in records:
+ nodes.update((record.pre_state, record.post_state))
+ edges[record.pre_state].add(record.post_state)
+ if record.terminal_success is True:
+ terminal_success.add(record.post_state)
+
+ distances = {node: math.inf for node in nodes}
+ queue: deque[str] = deque()
+ reverse: dict[str, set[str]] = defaultdict(set)
+ for source, targets in edges.items():
+ for target in targets:
+ reverse[target].add(source)
+ for node in terminal_success:
+ distances[node] = 0
+ queue.append(node)
+ while queue:
+ child = queue.popleft()
+ for parent in reverse.get(child, ()):
+ candidate = distances[child] + 1
+ if candidate < distances[parent]:
+ distances[parent] = candidate
+ queue.append(parent)
+ return distances
+
+
+def _score_graph(records: list[Transition], gamma: float = 0.1) -> list[TurnScore]:
+ distances = _graph_distances(records)
+ raw: list[tuple[Transition, float]] = []
+ for record in records:
+ distance = distances.get(record.post_state, math.inf)
+ graph_return = 0.0 if math.isinf(distance) else 10.0 * gamma**distance
+ raw.append((record, graph_return))
+
+ groups: dict[tuple[str, str], list[tuple[Transition, float]]] = defaultdict(list)
+ for record, graph_return in raw:
+ groups[(record.prompt_id, record.pre_state)].append((record, graph_return))
+
+ scores: list[TurnScore] = []
+ for (_, _), group in groups.items():
+ values = [value for _, value in group]
+ mean = statistics.fmean(values)
+ std = statistics.stdev(values) if len(values) > 1 else 0.0
+ for record, graph_return in group:
+ advantage = 0.0 if std == 0.0 else (graph_return - mean) / std
+ scores.append(
+ TurnScore(
+ pre_state=record.pre_state,
+ action=record.action,
+ action_correct=record.action_correct,
+ graph_return=graph_return,
+ graph_advantage=advantage,
+ ))
+ return scores
+
+
+def _ranking_accuracy(scores: list[TurnScore]) -> float:
+ groups: dict[str, list[TurnScore]] = defaultdict(list)
+ for score in scores:
+ groups[score.pre_state].append(score)
+ comparisons = 0
+ correct = 0
+ for group in groups.values():
+ good = [score.graph_advantage for score in group if score.action_correct]
+ bad = [score.graph_advantage for score in group if not score.action_correct]
+ if not good or not bad:
+ continue
+ comparisons += 1
+ correct += statistics.fmean(good) > statistics.fmean(bad)
+ return correct / comparisons if comparisons else 0.0
+
+
+def _sample_std(values: list[float]) -> float:
+ return statistics.stdev(values) if len(values) > 1 else 0.0
+
+
+def _episode_advantages(records: list[Transition]) -> dict[str, float]:
+ rollouts: dict[tuple[str, str], list[Transition]] = defaultdict(list)
+ for record in records:
+ rollouts[(record.prompt_id, record.rollout_id)].append(record)
+ by_prompt: dict[str, list[tuple[str, float]]] = defaultdict(list)
+ for (prompt_id, rollout_id), turns in rollouts.items():
+ reward = 1.0 if turns[-1].terminal_success is True else 0.0
+ by_prompt[prompt_id].append((rollout_id, reward))
+ advantages: dict[str, float] = {}
+ for values in by_prompt.values():
+ mean = statistics.fmean(reward for _, reward in values)
+ std = _sample_std([reward for _, reward in values])
+ for rollout_id, reward in values:
+ advantages[rollout_id] = 0.0 if std == 0.0 else (reward - mean) / std
+ return advantages
+
+
+def _implementation_probe(fixture: list[Transition]) -> dict:
+ from twinkle.advantage import StateGraphAdvantage
+
+ by_rollout: dict[tuple[str, str], list[Transition]] = defaultdict(list)
+ for record in fixture:
+ by_rollout[(record.prompt_id, record.rollout_id)].append(record)
+ trajectories = []
+ rewards = []
+ row_by_rollout = {}
+ prompt_ids = sorted({record.prompt_id for record in fixture})
+ for prompt_id in prompt_ids:
+ rollout_ids = sorted(rollout_id for candidate, rollout_id in by_rollout if candidate == prompt_id)
+ for rollout_id in rollout_ids:
+ turns = sorted(by_rollout[(prompt_id, rollout_id)], key=lambda record: record.turn_id)
+ row_by_rollout[(prompt_id, rollout_id)] = len(trajectories)
+ rewards.append(1.0 if turns[-1].terminal_success is True else 0.0)
+ trajectories.append({
+ 'labels': [0] * 16,
+ 'turn_credit_status': {
+ 'available': True,
+ 'fallback_reason': None,
+ },
+ 'turn_credits': [{
+ 'turn_id': turn.turn_id,
+ 'action_token_start': 1 + 2 * turn.turn_id,
+ 'action_token_end': 2 + 2 * turn.turn_id,
+ 'pre_state_key': turn.pre_state,
+ 'post_state_key': turn.post_state,
+ 'policy_version': 1,
+ 'terminal_success': turn.terminal_success,
+ } for turn in turns],
+ })
+
+ # Warm up lazy torch imports outside the measured graph path.
+ StateGraphAdvantage(beta_episode=0.0, beta_graph=0.0)(
+ rewards[:8],
+ trajectories[:8],
+ num_generations=8,
+ )
+ advantage = StateGraphAdvantage(beta_episode=0.0, beta_graph=1.0)
+ start = time.perf_counter()
+ values = advantage(rewards, trajectories, num_generations=8)
+ elapsed_ms = (time.perf_counter() - start) * 1000.0
+
+ state_groups: dict[tuple[str, str], list[tuple[float, bool]]] = defaultdict(list)
+ for record in fixture:
+ row = row_by_rollout[(record.prompt_id, record.rollout_id)]
+ position = 1 + 2 * record.turn_id
+ state_groups[(record.prompt_id, record.pre_state)].append(
+ (float(values[row, position].item()), record.action_correct))
+ comparisons = 0
+ correct = 0
+ for group in state_groups.values():
+ good = [value for value, is_correct in group if is_correct]
+ bad = [value for value, is_correct in group if not is_correct]
+ if not good or not bad:
+ continue
+ comparisons += 1
+ correct += statistics.fmean(good) > statistics.fmean(bad)
+
+ return {
+ 'credit_ranking_accuracy': correct / comparisons if comparisons else 0.0,
+ 'cpu_time_ms': elapsed_ms,
+ 'transition_count': advantage.last_diagnostics.transition_count,
+ 'comparable_state_group_count': advantage.last_diagnostics.comparable_state_group_count,
+ 'fallback_counts': advantage.last_diagnostics.fallback_counts,
+ }
+
+
+def run_gate0() -> dict:
+ fixture = _build_fixture()
+ start = time.perf_counter()
+ scores = _score_graph(fixture)
+ elapsed_ms = (time.perf_counter() - start) * 1000.0
+ implementation = _implementation_probe(fixture)
+
+ state_coverage = sum(bool(record.pre_state and record.post_state) for record in fixture) / len(fixture)
+ group_sizes: dict[tuple[str, str], int] = defaultdict(int)
+ for record in fixture:
+ group_sizes[(record.prompt_id, record.pre_state)] += 1
+ same_state_support = sum(size >= 2 for size in group_sizes.values()) / len(group_sizes)
+ rank_accuracy = _ranking_accuracy(scores)
+
+ fallback_advantage = _episode_advantages(fixture)
+ valid_rollout_ids = {record.rollout_id for record in fixture}
+ fallback_complete = valid_rollout_ids == set(fallback_advantage)
+
+ no_success = [
+ Transition(**{**asdict(record), 'terminal_success': None})
+ for record in fixture
+ if not record.post_state.endswith(':success')
+ ]
+ no_success_scores = _score_graph(no_success)
+ no_success_graph_mass = sum(abs(score.graph_advantage) for score in no_success_scores)
+
+ thresholds = {
+ 'state_key_coverage': 0.95,
+ 'same_state_support': 0.40,
+ 'credit_ranking_accuracy': 0.80,
+ 'fallback_rate_max': 0.05,
+ 'cpu_time_ms_max': 100.0,
+ }
+ observed = {
+ 'state_key_coverage': state_coverage,
+ 'same_state_support': same_state_support,
+ 'credit_ranking_accuracy': rank_accuracy,
+ 'fallback_rate': 0.0,
+ 'cpu_time_ms': elapsed_ms,
+ 'implementation_credit_ranking_accuracy': implementation['credit_ranking_accuracy'],
+ 'implementation_cpu_time_ms': implementation['cpu_time_ms'],
+ }
+ checks = {
+ 'state_key_coverage': state_coverage >= thresholds['state_key_coverage'],
+ 'same_state_support': same_state_support >= thresholds['same_state_support'],
+ 'credit_ranking_accuracy': rank_accuracy >= thresholds['credit_ranking_accuracy'],
+ 'fallback_rate': observed['fallback_rate'] <= thresholds['fallback_rate_max'],
+ 'cpu_time_ms': elapsed_ms <= thresholds['cpu_time_ms_max'],
+ 'implementation_credit_ranking_accuracy':
+ implementation['credit_ranking_accuracy'] >= thresholds['credit_ranking_accuracy'],
+ 'implementation_cpu_time_ms': implementation['cpu_time_ms'] <= thresholds['cpu_time_ms_max'],
+ 'implementation_transition_count': implementation['transition_count'] == len(fixture),
+ 'implementation_no_fallback': implementation['fallback_counts'] == {},
+ 'episode_fallback_complete': fallback_complete,
+ 'no_success_has_zero_graph_mass': no_success_graph_mass == 0.0,
+ }
+ return {
+ 'passed': all(checks.values()),
+ 'thresholds': thresholds,
+ 'observed': observed,
+ 'checks': checks,
+ 'fixture': {
+ 'transition_count': len(fixture),
+ 'state_group_count': len(group_sizes),
+ 'mean_group_support': statistics.fmean(group_sizes.values()),
+ },
+ 'negative_controls': {
+ 'no_success_graph_mass': no_success_graph_mass,
+ },
+ 'implementation': implementation,
+ 'extra_policy_forwards': 0,
+ }
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--output', type=Path)
+ args = parser.parse_args()
+ result = run_gate0()
+ text = json.dumps(result, indent=2, ensure_ascii=False)
+ print(text)
+ if args.output:
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ args.output.write_text(text + '\n', encoding='utf-8')
+ return 0 if result['passed'] else 1
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())
diff --git a/cookbook/rl/graphcredit/gate1.py b/cookbook/rl/graphcredit/gate1.py
new file mode 100644
index 000000000..694daa3bc
--- /dev/null
+++ b/cookbook/rl/graphcredit/gate1.py
@@ -0,0 +1,777 @@
+"""Matched-budget Gate 1 training for GRPO versus StateGraphAdvantage.
+
+The task is a deterministic two-step grid world. A wrong move ends the episode
+with reward 0, while reaching the goal returns 1. GRPO therefore only sees the
+final episode result. StateGraphAdvantage can additionally distinguish a useful
+intermediate move from a wrong move even when the episode later fails.
+
+This is a mechanism study, not a production training entry point. It uses a
+standalone Transformers + PEFT loop so both arms share the same rollout,
+policy-gradient update and token budget.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Sequence
+
+import numpy as np
+import torch
+
+from twinkle.advantage import GRPOAdvantage, StateGraphAdvantage
+
+
+ACTIONS = ('up', 'down', 'left', 'right')
+MOVES = {
+ 'up': (-1, 0),
+ 'down': (1, 0),
+ 'left': (0, -1),
+ 'right': (0, 1),
+}
+ACTION_PATTERN = re.compile(r'\s*(up|down|left|right)\s*', re.IGNORECASE)
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--model-path', required=True)
+ parser.add_argument('--mode', choices=('grpo', 'graphcredit'), required=True)
+ parser.add_argument('--output-dir', required=True)
+ parser.add_argument('--device', default='npu:0')
+ parser.add_argument('--seed', type=int, default=20261118)
+ parser.add_argument('--steps', type=int, default=20)
+ parser.add_argument('--eval-tasks', type=int, default=24)
+ parser.add_argument('--eval-every', type=int, default=0)
+ parser.add_argument('--group-size', type=int, default=8)
+ parser.add_argument('--distance', type=int, default=2)
+ parser.add_argument('--grid-size', type=int, default=5)
+ parser.add_argument('--max-new-tokens', type=int, default=24)
+ parser.add_argument('--max-length', type=int, default=1024)
+ parser.add_argument('--learning-rate', type=float, default=5e-5)
+ parser.add_argument('--lora-rank', type=int, default=8)
+ parser.add_argument('--lora-alpha', type=int, default=16)
+ parser.add_argument('--temperature', type=float, default=1.2)
+ parser.add_argument('--top-p', type=float, default=0.95)
+ parser.add_argument('--group-retries', type=int, default=12)
+ parser.add_argument('--max-step-attempts', type=int, default=6)
+ parser.add_argument('--beta-episode', type=float, default=1.0)
+ parser.add_argument('--beta-graph', type=float, default=1.0)
+ parser.add_argument('--gamma', type=float, default=0.5)
+ parser.add_argument('--min-state-support', type=int, default=2)
+ parser.add_argument('--max-grad-norm', type=float, default=1.0)
+ return parser.parse_args()
+
+
+def seed_everything(seed: int) -> None:
+ random.seed(seed)
+ np.random.seed(seed)
+ torch.manual_seed(seed)
+ if hasattr(torch, 'npu'):
+ torch.npu.manual_seed_all(seed)
+
+
+def ensure_npu(device_name: str) -> None:
+ if device_name.startswith('npu'):
+ import torch_npu # noqa: F401
+
+
+def reset_peak_hbm() -> None:
+ if hasattr(torch, 'npu') and torch.npu.is_available():
+ torch.npu.reset_peak_memory_stats()
+ elif torch.cuda.is_available():
+ torch.cuda.reset_peak_memory_stats()
+
+
+def peak_hbm_gb() -> float:
+ if hasattr(torch, 'npu') and torch.npu.is_available():
+ return float(torch.npu.max_memory_allocated() / 2**30)
+ if torch.cuda.is_available():
+ return float(torch.cuda.max_memory_allocated() / 2**30)
+ return 0.0
+
+
+def synchronize(device: torch.device) -> None:
+ if device.type == 'npu':
+ torch.npu.synchronize()
+ elif device.type == 'cuda':
+ torch.cuda.synchronize()
+
+
+@dataclass(frozen=True)
+class GridTask:
+ task_uid: int
+ grid_size: int
+ distance: int
+ start: tuple[int, int]
+ goal: tuple[int, int]
+
+ @property
+ def task_key(self) -> str:
+ return f'task-{self.task_uid}'
+
+ def state_key(self, position: tuple[int, int]) -> str:
+ return f'{self.task_key}:{position[0]}:{position[1]}'
+
+ @property
+ def dead_state_key(self) -> str:
+ return f'{self.task_key}:dead'
+
+
+def make_task(task_uid: int, *, grid_size: int, distance: int) -> GridTask:
+ if distance < 1:
+ raise ValueError('distance must be positive')
+ if grid_size <= distance:
+ raise ValueError('grid_size must be larger than distance')
+ rng = random.Random(task_uid)
+ for _ in range(10_000):
+ start = (rng.randrange(grid_size), rng.randrange(grid_size))
+ axis = rng.choice(('row', 'col'))
+ sign = rng.choice((-1, 1))
+ goal = (
+ start[0] + sign * distance if axis == 'row' else start[0],
+ start[1] + sign * distance if axis == 'col' else start[1],
+ )
+ if 0 <= goal[0] < grid_size and 0 <= goal[1] < grid_size:
+ return GridTask(task_uid, grid_size, distance, start, goal)
+ raise RuntimeError(f'could not generate task {task_uid}')
+
+
+def format_action(action: str) -> str:
+ return f'{action}'
+
+
+def parse_action(text: str) -> str | None:
+ matches = ACTION_PATTERN.findall(text)
+ if len(matches) == 1:
+ return matches[0].lower()
+ stripped = text.strip().lower()
+ return stripped if stripped in ACTIONS else None
+
+
+def render_prompt(task: GridTask, position: tuple[int, int], history: Sequence[str]) -> str:
+ row, col = position
+ goal_row, goal_col = task.goal
+ distance = abs(goal_row - row) + abs(goal_col - col)
+ history_text = ' -> '.join(history) if history else ''
+ return (
+ 'You control a robot on a rectangular grid. Do not explain or reason aloud. '
+ 'Reply with exactly one action tag: up, down, '
+ 'left, or right.\n'
+ 'Coordinates are zero-based (row, column). Up decreases row, down increases row, '
+ 'left decreases column and right increases column. A move that does not reduce '
+ 'the Manhattan distance to the goal immediately ends the episode.\n'
+ f'Grid size: {task.grid_size}x{task.grid_size}.\n'
+ f'Current position: ({row}, {col}).\n'
+ f'Goal position: ({goal_row}, {goal_col}).\n'
+ f'Remaining Manhattan distance: {distance}.\n'
+ f'Action history: {history_text}.\n'
+ 'Choose the next action.'
+ )
+
+
+def render_messages(prompt: str) -> list[dict[str, str]]:
+ return [
+ {
+ 'role': 'system',
+ 'content': 'You are a precise grid-navigation policy.'
+ },
+ {
+ 'role': 'user',
+ 'content': prompt
+ },
+ ]
+
+
+@dataclass
+class EpisodeTurn:
+ prompt: str
+ action: str
+ pre_state_key: str
+ post_state_key: str
+ action_token_start: int
+ action_token_end: int
+ terminal_success: bool | None
+
+
+@dataclass
+class Episode:
+ task: GridTask
+ uid: str
+ position: tuple[int, int]
+ done: bool = False
+ success: bool = False
+ reward: float = 0.0
+ invalid_action: bool = False
+ input_ids: list[int] = field(default_factory=list)
+ turns: list[EpisodeTurn] = field(default_factory=list)
+
+ @property
+ def history(self) -> list[str]:
+ return [turn.action for turn in self.turns]
+
+ def to_trajectory(self, policy_version: int) -> dict[str, Any]:
+ records = [
+ {
+ 'turn_id': index,
+ 'action_token_start': turn.action_token_start,
+ 'action_token_end': turn.action_token_end,
+ 'observation_token_start': None,
+ 'observation_token_end': None,
+ 'pre_state_key': turn.pre_state_key,
+ 'post_state_key': turn.post_state_key,
+ 'tool_events': [],
+ 'policy_version': policy_version,
+ 'terminal_success': turn.terminal_success,
+ }
+ for index, turn in enumerate(self.turns)
+ ]
+ return {
+ 'input_ids': list(self.input_ids),
+ 'turn_credits': records,
+ 'turn_credit_status': {
+ 'available': bool(records) and not self.invalid_action,
+ 'fallback_reason': None if records else 'turn_credit_empty',
+ },
+ 'episode_reward': self.reward,
+ 'terminal_success': self.success,
+ }
+
+
+def step_episode(episode: Episode, action: str) -> None:
+ if episode.done:
+ raise RuntimeError('cannot step a finished episode')
+ if action not in MOVES:
+ raise ValueError(f'unsupported action: {action}')
+
+ row, col = episode.position
+ delta_row, delta_col = MOVES[action]
+ next_position = (row + delta_row, col + delta_col)
+ current_distance = abs(row - episode.task.goal[0]) + abs(col - episode.task.goal[1])
+ next_distance = (
+ abs(next_position[0] - episode.task.goal[0])
+ + abs(next_position[1] - episode.task.goal[1])
+ )
+ out_of_bounds = not (
+ 0 <= next_position[0] < episode.task.grid_size
+ and 0 <= next_position[1] < episode.task.grid_size
+ )
+ if out_of_bounds or next_distance >= current_distance:
+ episode.done = True
+ episode.success = False
+ episode.reward = 0.0
+ episode.position = next_position
+ return
+
+ episode.position = next_position
+ if next_position == episode.task.goal:
+ episode.done = True
+ episode.success = True
+ episode.reward = 1.0
+
+
+def _append_prompt_action(
+ episode: Episode,
+ prompt_ids: Sequence[int],
+ action_ids: Sequence[int],
+ *,
+ eos_token_id: int | None,
+) -> tuple[int, int]:
+ before = len(episode.input_ids)
+ episode.input_ids.extend(int(token) for token in prompt_ids)
+ action_start = before + len(prompt_ids) - 1
+ episode.input_ids.extend(int(token) for token in action_ids)
+ action_end = action_start + len(action_ids)
+ if eos_token_id is not None:
+ episode.input_ids.append(int(eos_token_id))
+ return action_start, action_end
+
+
+def rollout_group(
+ policy: Any,
+ tokenizer: Any,
+ task: GridTask,
+ *,
+ group_size: int,
+ max_new_tokens: int,
+ max_length: int,
+ temperature: float,
+ top_p: float,
+ device: torch.device,
+ sample: bool,
+) -> list[Episode]:
+ episodes = [
+ Episode(task=task, uid=f'{task.task_uid}:{index}', position=task.start)
+ for index in range(group_size)
+ ]
+ for _turn in range(1, task.distance + 1):
+ active = [episode for episode in episodes if not episode.done]
+ if not active:
+ break
+ prompts = [render_prompt(task, episode.position, episode.history) for episode in active]
+ outputs = generate_actions(
+ policy,
+ tokenizer,
+ prompts,
+ device,
+ sample=sample,
+ max_new_tokens=max_new_tokens,
+ temperature=temperature,
+ top_p=top_p,
+ )
+ for episode, prompt, raw_output in zip(active, prompts, outputs):
+ action = parse_action(raw_output)
+ if action is None:
+ episode.invalid_action = True
+ episode.done = True
+ episode.reward = 0.0
+ continue
+ prompt_text = render_chat_prompt(tokenizer, prompt)
+ prompt_ids = encode_text(tokenizer, prompt_text)
+ action_ids = encode_text(tokenizer, format_action(action))
+ if len(episode.input_ids) + len(prompt_ids) + len(action_ids) > max_length:
+ episode.invalid_action = True
+ episode.done = True
+ episode.reward = 0.0
+ continue
+ pre_state_key = task.state_key(episode.position)
+ action_start, action_end = _append_prompt_action(
+ episode,
+ prompt_ids,
+ action_ids,
+ eos_token_id=tokenizer.eos_token_id,
+ )
+ step_episode(episode, action)
+ post_state_key = (
+ task.state_key(episode.position) if episode.success or not episode.done else task.dead_state_key
+ )
+ episode.turns.append(
+ EpisodeTurn(
+ prompt=prompt,
+ action=action,
+ pre_state_key=pre_state_key,
+ post_state_key=post_state_key,
+ action_token_start=action_start,
+ action_token_end=action_end,
+ terminal_success=True if episode.success else (False if episode.done else None),
+ ))
+ for episode in episodes:
+ if not episode.done:
+ episode.done = True
+ episode.reward = 0.0
+ return episodes
+
+
+def render_chat_prompt(tokenizer: Any, prompt: str) -> str:
+ return tokenizer.apply_chat_template(
+ render_messages(prompt),
+ tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False,
+ )
+
+
+def encode_text(tokenizer: Any, text: str) -> list[int]:
+ return list(tokenizer(text, add_special_tokens=False, truncation=False)['input_ids'])
+
+
+def generate_actions(
+ policy: Any,
+ tokenizer: Any,
+ prompts: Sequence[str],
+ device: torch.device,
+ *,
+ sample: bool,
+ max_new_tokens: int,
+ temperature: float,
+ top_p: float,
+) -> list[str]:
+ rendered = [render_chat_prompt(tokenizer, prompt) for prompt in prompts]
+ encoded = tokenizer(
+ rendered,
+ padding=True,
+ return_tensors='pt',
+ add_special_tokens=False,
+ )
+ encoded = {key: value.to(device) for key, value in encoded.items()}
+ policy.eval()
+ policy.config.use_cache = True
+ kwargs: dict[str, Any] = {
+ 'max_new_tokens': max_new_tokens,
+ 'do_sample': sample,
+ 'pad_token_id': tokenizer.pad_token_id,
+ 'eos_token_id': tokenizer.eos_token_id,
+ 'use_cache': True,
+ }
+ if sample:
+ kwargs.update(temperature=temperature, top_p=top_p)
+ with torch.inference_mode():
+ generated = policy.generate(**encoded, **kwargs)
+ width = encoded['input_ids'].shape[1]
+ return tokenizer.batch_decode(generated[:, width:], skip_special_tokens=True)
+
+
+def compute_advantages(
+ mode: str,
+ episodes: Sequence[Episode],
+ *,
+ policy_version: int,
+ group_size: int,
+ args: argparse.Namespace,
+) -> tuple[torch.Tensor, dict[str, Any]]:
+ rewards = [episode.reward for episode in episodes]
+ trajectories = [episode.to_trajectory(policy_version) for episode in episodes]
+ max_length = max((len(episode.input_ids) for episode in episodes), default=0)
+ if mode == 'grpo':
+ episode_advantage = GRPOAdvantage()(rewards, num_generations=group_size, scale='group')
+ advantages = episode_advantage.unsqueeze(1).repeat(1, max_length)
+ diagnostics = {'mode': 'grpo'}
+ return advantages, diagnostics
+
+ graph = StateGraphAdvantage(
+ beta_episode=args.beta_episode,
+ beta_graph=args.beta_graph,
+ gamma=args.gamma,
+ min_state_support=args.min_state_support,
+ )
+ advantages = graph(rewards, trajectories, num_generations=group_size)
+ diagnostics = {
+ 'mode': 'graphcredit',
+ 'transition_count': graph.last_diagnostics.transition_count,
+ 'graph_node_count': graph.last_diagnostics.graph_node_count,
+ 'graph_edge_count': graph.last_diagnostics.graph_edge_count,
+ 'comparable_state_group_count': graph.last_diagnostics.comparable_state_group_count,
+ 'fallback_counts': graph.last_diagnostics.fallback_counts,
+ }
+ return advantages, diagnostics
+
+
+def pad_episode_batch(
+ episodes: Sequence[Episode],
+ pad_token_id: int,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ max_length = max(len(episode.input_ids) for episode in episodes)
+ rows = []
+ masks = []
+ for episode in episodes:
+ padding = max_length - len(episode.input_ids)
+ rows.append(episode.input_ids + [pad_token_id] * padding)
+ masks.append([1] * len(episode.input_ids) + [0] * padding)
+ return (
+ torch.tensor(rows, dtype=torch.long),
+ torch.tensor(masks, dtype=torch.long),
+ )
+
+
+def policy_gradient_update(
+ policy: Any,
+ optimizer: torch.optim.Optimizer,
+ episodes: Sequence[Episode],
+ advantages: torch.Tensor,
+ *,
+ device: torch.device,
+ pad_token_id: int,
+ max_grad_norm: float,
+) -> tuple[float, int]:
+ policy.train()
+ policy.config.use_cache = False
+ optimizer.zero_grad(set_to_none=True)
+ input_ids, attention_mask = pad_episode_batch(episodes, pad_token_id)
+ input_ids = input_ids.to(device)
+ attention_mask = attention_mask.to(device)
+ outputs = policy(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ use_cache=False,
+ )
+ logits = outputs.logits[:, :-1, :].float()
+ targets = input_ids[:, 1:]
+ token_logps = torch.log_softmax(logits, dim=-1).gather(
+ -1,
+ targets.unsqueeze(-1),
+ ).squeeze(-1)
+ advantages = advantages.to(device=device, dtype=token_logps.dtype)
+
+ loss = input_ids.new_zeros((), dtype=torch.float32)
+ action_tokens = 0
+ for row, episode in enumerate(episodes):
+ for turn in episode.turns:
+ start = turn.action_token_start
+ end = turn.action_token_end
+ if start < 0 or end <= start or end > token_logps.shape[1]:
+ raise RuntimeError(
+ f'invalid action span for {episode.uid}: [{start}, {end}) '
+ f'with sequence width {token_logps.shape[1]}')
+ loss = loss - (
+ token_logps[row, start:end] * advantages[row, start:end]
+ ).sum()
+ action_tokens += end - start
+ if action_tokens == 0:
+ raise RuntimeError('episode group contains no trainable action tokens')
+ loss = loss / action_tokens
+ loss.backward()
+ torch.nn.utils.clip_grad_norm_(
+ [parameter for parameter in policy.parameters() if parameter.requires_grad],
+ max_grad_norm,
+ )
+ optimizer.step()
+ return float(loss.detach().cpu()), action_tokens
+
+
+def evaluate(
+ policy: Any,
+ tokenizer: Any,
+ args: argparse.Namespace,
+ device: torch.device,
+ *,
+ split_offset: int,
+) -> dict[str, float]:
+ successes = 0
+ invalid = 0
+ turns = 0
+ rewards = []
+ started = time.perf_counter()
+ for index in range(args.eval_tasks):
+ task = make_task(
+ split_offset + index,
+ grid_size=args.grid_size,
+ distance=args.distance,
+ )
+ episode = rollout_group(
+ policy,
+ tokenizer,
+ task,
+ group_size=1,
+ max_new_tokens=args.max_new_tokens,
+ max_length=args.max_length,
+ temperature=args.temperature,
+ top_p=args.top_p,
+ device=device,
+ sample=False,
+ )[0]
+ successes += int(episode.success)
+ invalid += int(episode.invalid_action)
+ turns += len(episode.turns)
+ rewards.append(episode.reward)
+ total = max(1, len(rewards))
+ return {
+ 'success_rate': successes / total,
+ 'invalid_rate': invalid / total,
+ 'reward_mean': float(np.mean(rewards)) if rewards else 0.0,
+ 'mean_turns': turns / total,
+ 'eval_seconds': time.perf_counter() - started,
+ }
+
+
+def sample_training_group(
+ policy: Any,
+ tokenizer: Any,
+ args: argparse.Namespace,
+ device: torch.device,
+ *,
+ step: int,
+) -> tuple[GridTask, list[Episode]] | None:
+ for attempt in range(args.group_retries):
+ task_uid = args.seed + 10_000 + step * 100 + attempt
+ task = make_task(task_uid, grid_size=args.grid_size, distance=args.distance)
+ episodes = rollout_group(
+ policy,
+ tokenizer,
+ task,
+ group_size=args.group_size,
+ max_new_tokens=args.max_new_tokens,
+ max_length=args.max_length,
+ temperature=args.temperature,
+ top_p=args.top_p,
+ device=device,
+ sample=True,
+ )
+ if any(episode.invalid_action for episode in episodes):
+ continue
+ rewards = [episode.reward for episode in episodes]
+ if not any(rewards) or all(rewards):
+ continue
+ return task, episodes
+ return None
+
+
+def load_tokenizer(model_path: str) -> Any:
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+ if tokenizer.pad_token is None:
+ tokenizer.pad_token = tokenizer.eos_token
+ tokenizer.padding_side = 'left'
+ return tokenizer
+
+
+def load_policy(model_path: str, args: argparse.Namespace, device: torch.device) -> Any:
+ from peft import LoraConfig, get_peft_model
+ from transformers import AutoModelForCausalLM
+
+ policy = AutoModelForCausalLM.from_pretrained(
+ model_path,
+ dtype=torch.bfloat16,
+ trust_remote_code=True,
+ low_cpu_mem_usage=True,
+ ).to(device)
+ if hasattr(policy, 'gradient_checkpointing_enable'):
+ policy.gradient_checkpointing_enable()
+ policy = get_peft_model(
+ policy,
+ LoraConfig(
+ task_type='CAUSAL_LM',
+ r=args.lora_rank,
+ lora_alpha=args.lora_alpha,
+ lora_dropout=0.05,
+ target_modules=[
+ 'q_proj',
+ 'k_proj',
+ 'v_proj',
+ 'o_proj',
+ 'gate_proj',
+ 'up_proj',
+ 'down_proj',
+ ],
+ ),
+ )
+ policy.config.use_cache = False
+ return policy
+
+
+def train(policy: Any, tokenizer: Any, args: argparse.Namespace, device: torch.device) -> dict[str, Any]:
+ trainable = [parameter for parameter in policy.parameters() if parameter.requires_grad]
+ optimizer = torch.optim.AdamW(trainable, lr=args.learning_rate)
+ initial_eval = evaluate(
+ policy,
+ tokenizer,
+ args,
+ device,
+ split_offset=args.seed + 500_000,
+ )
+ reset_peak_hbm()
+ history = []
+ completed_steps = 0
+ attempted_steps = 0
+ max_attempts = args.steps * args.max_step_attempts
+ while completed_steps < args.steps and attempted_steps < max_attempts:
+ attempted_steps += 1
+ step_started = time.perf_counter()
+ sampled = sample_training_group(
+ policy,
+ tokenizer,
+ args,
+ device,
+ step=attempted_steps - 1,
+ )
+ if sampled is None:
+ history.append({
+ 'attempt': attempted_steps,
+ 'completed_steps': completed_steps,
+ 'skipped': True,
+ })
+ continue
+ task, episodes = sampled
+ synchronize(device)
+ rollout_seconds = time.perf_counter() - step_started
+ rewards = [episode.reward for episode in episodes]
+ advantages, diagnostics = compute_advantages(
+ args.mode,
+ episodes,
+ policy_version=completed_steps + 1,
+ group_size=args.group_size,
+ args=args,
+ )
+ loss, action_tokens = policy_gradient_update(
+ policy,
+ optimizer,
+ episodes,
+ advantages,
+ device=device,
+ pad_token_id=tokenizer.pad_token_id,
+ max_grad_norm=args.max_grad_norm,
+ )
+ synchronize(device)
+ record = {
+ 'step': completed_steps + 1,
+ 'attempt': attempted_steps,
+ 'task_uid': task.task_uid,
+ 'reward_mean': float(np.mean(rewards)),
+ 'success_rate': float(np.mean([episode.success for episode in episodes])),
+ 'mean_actions': float(np.mean([len(episode.turns) for episode in episodes])),
+ 'loss': loss,
+ 'action_tokens': action_tokens,
+ 'rollout_seconds': rollout_seconds,
+ 'step_seconds': time.perf_counter() - step_started,
+ 'peak_hbm_gb': peak_hbm_gb(),
+ **diagnostics,
+ }
+ history.append(record)
+ print(json.dumps(record, ensure_ascii=False), flush=True)
+ completed_steps += 1
+ if args.eval_every > 0 and completed_steps % args.eval_every == 0:
+ heldout = evaluate(
+ policy,
+ tokenizer,
+ args,
+ device,
+ split_offset=args.seed + 500_000,
+ )
+ record['heldout_eval'] = heldout
+ print(
+ json.dumps(
+ {
+ 'step': completed_steps,
+ 'heldout_success_rate': heldout['success_rate'],
+ 'heldout_invalid_rate': heldout['invalid_rate'],
+ },
+ ensure_ascii=False,
+ ),
+ flush=True,
+ )
+ final_eval = evaluate(
+ policy,
+ tokenizer,
+ args,
+ device,
+ split_offset=args.seed + 500_000,
+ )
+ return {
+ 'mode': args.mode,
+ 'history': history,
+ 'completed_steps': completed_steps,
+ 'attempted_steps': attempted_steps,
+ 'skipped_steps': attempted_steps - completed_steps,
+ 'initial_eval': initial_eval,
+ 'final_eval': final_eval,
+ 'peak_hbm_gb': peak_hbm_gb(),
+ }
+
+
+def main() -> None:
+ args = parse_args()
+ ensure_npu(args.device)
+ seed_everything(args.seed)
+ device = torch.device(args.device)
+ output_dir = Path(args.output_dir)
+ output_dir.mkdir(parents=True, exist_ok=True)
+ tokenizer = load_tokenizer(args.model_path)
+ policy = load_policy(args.model_path, args, device)
+ started = time.perf_counter()
+ result = train(policy, tokenizer, args, device)
+ result['wall_seconds'] = time.perf_counter() - started
+ result['args'] = vars(args)
+ policy.save_pretrained(output_dir / 'adapter')
+ (output_dir / 'result.json').write_text(
+ json.dumps(result, indent=2, ensure_ascii=False),
+ encoding='utf-8',
+ )
+ print(json.dumps(result, indent=2, ensure_ascii=False), flush=True)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/cookbook/rl/graphcredit/run_gate1_a2.sh b/cookbook/rl/graphcredit/run_gate1_a2.sh
new file mode 100644
index 000000000..002fff9fb
--- /dev/null
+++ b/cookbook/rl/graphcredit/run_gate1_a2.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+MODE="${1:?usage: run_gate1_a2.sh [steps] [eval-tasks] [output-dir]}"
+STEPS="${2:-20}"
+EVAL_TASKS="${3:-24}"
+OUTPUT_DIR="${4:-outputs/graphcredit-gate1/${MODE}}"
+
+MODEL_PATH="${MODEL_PATH:-/workspace/shared_assets/models/Qwen/Qwen3-0.6B}"
+SEED="${SEED:-20261118}"
+GROUP_SIZE="${GROUP_SIZE:-8}"
+MAX_NEW_TOKENS="${MAX_NEW_TOKENS:-24}"
+MAX_LENGTH="${MAX_LENGTH:-1024}"
+LEARNING_RATE="${LEARNING_RATE:-5e-5}"
+DISTANCE="${DISTANCE:-2}"
+GRID_SIZE="${GRID_SIZE:-5}"
+EVAL_EVERY="${EVAL_EVERY:-0}"
+MAX_STEP_ATTEMPTS="${MAX_STEP_ATTEMPTS:-6}"
+
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+set -u
+export PYTHONPATH="/workspace/graphcredit-gate0/src:/workspace/graphcredit-deps${PYTHONPATH:+:${PYTHONPATH}}"
+
+cd /workspace/graphcredit-gate0
+exec /workspace/npu-ai-env/bin/python cookbook/rl/graphcredit/gate1.py \
+ --model-path "${MODEL_PATH}" \
+ --mode "${MODE}" \
+ --steps "${STEPS}" \
+ --eval-tasks "${EVAL_TASKS}" \
+ --eval-every "${EVAL_EVERY}" \
+ --group-size "${GROUP_SIZE}" \
+ --max-step-attempts "${MAX_STEP_ATTEMPTS}" \
+ --distance "${DISTANCE}" \
+ --grid-size "${GRID_SIZE}" \
+ --max-new-tokens "${MAX_NEW_TOKENS}" \
+ --max-length "${MAX_LENGTH}" \
+ --learning-rate "${LEARNING_RATE}" \
+ --seed "${SEED}" \
+ --device npu:0 \
+ --output-dir "${OUTPUT_DIR}"
diff --git "a/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/StateGraphAdvantage.md" "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/StateGraphAdvantage.md"
new file mode 100644
index 000000000..681645656
--- /dev/null
+++ "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/StateGraphAdvantage.md"
@@ -0,0 +1,72 @@
+# StateGraphAdvantage
+
+`StateGraphAdvantage` 在 episode-level GRPO advantage 之上,增加基于显式状态图的
+turn-level advantage。它依赖阶段 A 产生的 `turn_credits`,不根据聊天文本推断状态,
+也不会增加额外 policy forward。
+
+## 使用示例
+
+```python
+from twinkle.advantage import StateGraphAdvantage
+
+advantage_fn = StateGraphAdvantage(
+ beta_episode=1.0,
+ beta_graph=1.0,
+ gamma=0.1,
+ min_state_support=2,
+)
+
+advantages = advantage_fn(
+ rewards,
+ trajectories,
+ num_generations=4,
+)
+```
+
+每条 trajectory 需要包含:
+
+- `turn_credit_status.available=True`;
+- `turn_credits` 中每个 turn 的 `pre_state_key`、`post_state_key`;
+- 左闭右开的 `action_token_start` / `action_token_end`;
+- 明确的 `policy_version`;
+- 至少一个带 `terminal_success=True` 的状态边。
+
+## 计算方式
+
+同一 prompt group 内,状态键构成图节点,turn 构成有向边。成功终态的距离为 0,
+算法从成功态反向计算最短距离,并按相同 `pre_state_key` 对动作做归一化:
+
+```text
+R_t^G = 10 * gamma ^ distance(post_state_t)
+A_t^G = Normalize(R_t^G | same prompt group, same pre_state_key)
+A_t = beta_episode * A_t^E + beta_graph * A_t^G
+```
+
+`A_t^E` 每条 episode 只计算一次,然后广播到该 episode 的可训练 token。
+graph advantage 只写入 turn 的 action span,不覆盖 observation、bridge 或 padding。
+
+## 严格回退
+
+以下情况不会部分解释 graph advantage,而是保留原 GRPO 数值:
+
+- 缺失状态键或 terminal success;
+- 同一轮工具结果的状态或成功标记冲突;
+- action span、observation span 越界;
+- policy version 缺失、冲突或类型非法;
+- 没有成功终态;
+- 状态图节点或边超过预算;
+- turn credit 不可用或为空。
+
+回退原因记录在 `advantage_fn.last_diagnostics.fallback_counts`。相同状态的动作
+少于 `min_state_support` 时,该状态组保持零 graph advantage。
+
+## Gate 0
+
+确定性探针位于 `cookbook/rl/graphcredit/gate0.py`:
+
+```bash
+PYTHONPATH=src python cookbook/rl/graphcredit/gate0.py
+```
+
+探针检查状态覆盖率、同状态支撑、动作排序、无成功终态负例、回退完整性、
+CPU 计算耗时以及额外 policy forward 数量。
diff --git "a/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst" "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst"
index f1522152d..138837809 100644
--- "a/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst"
+++ "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst"
@@ -7,3 +7,4 @@
GRPOAdvantage.md
GroupAdmissionPolicy.md
RLOOAdvantage.md
+ StateGraphAdvantage.md
diff --git a/src/twinkle/advantage/__init__.py b/src/twinkle/advantage/__init__.py
index 519dd1fe4..72ded0894 100644
--- a/src/twinkle/advantage/__init__.py
+++ b/src/twinkle/advantage/__init__.py
@@ -3,10 +3,12 @@
from .gae import GAEAdvantage
from .grpo import GRPOAdvantage
from .rloo import RLOOAdvantage
+from .state_graph import StateGraphAdvantage
__all__ = [
'Advantage',
'GAEAdvantage',
'GRPOAdvantage',
'RLOOAdvantage',
+ 'StateGraphAdvantage',
]
diff --git a/src/twinkle/advantage/state_graph.py b/src/twinkle/advantage/state_graph.py
new file mode 100644
index 000000000..d4515bd11
--- /dev/null
+++ b/src/twinkle/advantage/state_graph.py
@@ -0,0 +1,284 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""State-graph turn credit with episode-level GRPO as the safe fallback."""
+
+from __future__ import annotations
+
+import math
+import statistics
+from collections import defaultdict, deque
+from dataclasses import dataclass, field
+from typing import Any, Mapping, Sequence
+
+from .base import Advantage
+from .grpo import GRPOAdvantage
+
+__all__ = ['StateGraphAdvantage', 'StateGraphAdvantageDiagnostics']
+
+
+@dataclass
+class StateGraphAdvantageDiagnostics:
+ transition_count: int = 0
+ graph_node_count: int = 0
+ graph_edge_count: int = 0
+ comparable_state_group_count: int = 0
+ fallback_counts: dict[str, int] = field(default_factory=dict)
+ output_shape: tuple[int, int] = (0, 0)
+
+ def add_fallback(self, reason: str) -> None:
+ self.fallback_counts[reason] = self.fallback_counts.get(reason, 0) + 1
+
+
+class StateGraphAdvantage(Advantage):
+ """Combine GRPO episode advantages with state-conditioned graph returns.
+
+ ``turn_credits`` are emitted by ``MultiTurnRollout`` and use labels-space,
+ half-open action spans. A group is graph-scored only when every participating
+ policy version matches, at least one explicit success state exists, and the
+ graph stays within the configured budget. Otherwise its rows remain exactly
+ equal to the episode-level GRPO advantage.
+ """
+
+ def __init__(
+ self,
+ *,
+ beta_episode: float = 1.0,
+ beta_graph: float = 1.0,
+ gamma: float = 0.1,
+ min_state_support: int = 2,
+ max_graph_nodes: int = 4096,
+ max_graph_edges: int = 16384,
+ epsilon: float = 1e-8,
+ ):
+ if not math.isfinite(beta_episode) or beta_episode < 0:
+ raise ValueError('beta_episode must be finite and non-negative')
+ if not math.isfinite(beta_graph) or beta_graph < 0:
+ raise ValueError('beta_graph must be finite and non-negative')
+ if not 0 < gamma <= 1:
+ raise ValueError('gamma must be in (0, 1]')
+ if min_state_support < 2:
+ raise ValueError('min_state_support must be at least two')
+ if max_graph_nodes < 1 or max_graph_edges < 1:
+ raise ValueError('graph budgets must be positive')
+ if epsilon <= 0:
+ raise ValueError('epsilon must be positive')
+ self.beta_episode = beta_episode
+ self.beta_graph = beta_graph
+ self.gamma = gamma
+ self.min_state_support = min_state_support
+ self.max_graph_nodes = max_graph_nodes
+ self.max_graph_edges = max_graph_edges
+ self.epsilon = epsilon
+ self.last_diagnostics: StateGraphAdvantageDiagnostics | None = None
+
+ def compute(
+ self,
+ rewards: Sequence[float],
+ trajectories: Sequence[Mapping[str, Any]],
+ *,
+ num_generations: int,
+ sequence_lengths: Sequence[int] | None = None,
+ ) -> tuple[Any, StateGraphAdvantageDiagnostics]:
+ import torch
+
+ if num_generations <= 1:
+ raise ValueError('StateGraphAdvantage requires at least two generations per prompt')
+ if len(trajectories) != len(rewards) or len(rewards) % num_generations:
+ raise ValueError('rewards and trajectories must contain complete groups')
+ if sequence_lengths is None:
+ lengths = [self._sequence_length(trajectory) for trajectory in trajectories]
+ else:
+ lengths = [int(length) for length in sequence_lengths]
+ if len(lengths) != len(trajectories) or any(length <= 0 for length in lengths):
+ raise ValueError('sequence_lengths must be positive and match trajectories')
+
+ episode = GRPOAdvantage()(torch.as_tensor(rewards, dtype=torch.float32), num_generations=num_generations)
+ advantages = (self.beta_episode * episode).unsqueeze(1).repeat(1, max(lengths))
+ diagnostics = StateGraphAdvantageDiagnostics(output_shape=tuple(advantages.shape))
+
+ for group_start in range(0, len(trajectories), num_generations):
+ group_rows = range(group_start, group_start + num_generations)
+ records = []
+ fallback_rows: set[int] = set()
+ versions: set[int | str] = set()
+ versioned_records = 0
+ for row in group_rows:
+ trajectory = trajectories[row]
+ status = trajectory.get('turn_credit_status') or {}
+ if status.get('available') is not True:
+ fallback_rows.add(row)
+ diagnostics.add_fallback(str(status.get('fallback_reason') or 'turn_credit_unavailable'))
+ continue
+ credits = list(trajectory.get('turn_credits') or [])
+ if not credits:
+ fallback_rows.add(row)
+ diagnostics.add_fallback('turn_credit_empty')
+ continue
+ for record in credits:
+ try:
+ parsed = self._validate_record(record, lengths[row])
+ except (TypeError, ValueError) as exc:
+ fallback_rows.add(row)
+ diagnostics.add_fallback(str(exc) or 'turn_credit_invalid')
+ break
+ records.append((row, parsed))
+ if parsed.get('policy_version') is not None:
+ versions.add(parsed['policy_version'])
+ versioned_records += 1
+
+ if len(versions) > 1:
+ diagnostics.add_fallback('policy_version_mismatch')
+ continue
+ if not records:
+ continue
+ if versioned_records != len(records):
+ diagnostics.add_fallback('policy_version_missing')
+ continue
+
+ valid_records = [(row, record) for row, record in records if row not in fallback_rows]
+ if not valid_records:
+ continue
+ try:
+ distances, node_count, edge_count = self._distances(valid_records)
+ except ValueError as exc:
+ diagnostics.add_fallback(str(exc))
+ continue
+ diagnostics.graph_node_count += node_count
+ diagnostics.graph_edge_count += edge_count
+ diagnostics.transition_count += len(valid_records)
+
+ state_groups: dict[str, list[tuple[int, Mapping[str, Any], float]]] = defaultdict(list)
+ for row, record in valid_records:
+ distance = distances.get(record['post_state_key'], math.inf)
+ graph_return = 0.0 if math.isinf(distance) else 10.0 * self.gamma**distance
+ state_groups[record['pre_state_key']].append((row, record, graph_return))
+
+ for group in state_groups.values():
+ if len(group) < self.min_state_support:
+ continue
+ values = [value for _, _, value in group]
+ mean = statistics.fmean(values)
+ std = statistics.stdev(values)
+ if std <= self.epsilon:
+ continue
+ diagnostics.comparable_state_group_count += 1
+ for row, record, value in group:
+ local = (value - mean) / std
+ start, end = record['action_token_start'], record['action_token_end']
+ advantages[row, start:end] += self.beta_graph * local
+
+ return advantages, diagnostics
+
+ @staticmethod
+ def _sequence_length(trajectory: Mapping[str, Any]) -> int:
+ for key in ('labels', 'input_ids', 'completion_mask'):
+ value = trajectory.get(key)
+ if value is not None:
+ return len(value)
+ raise ValueError('trajectory needs labels, input_ids, or completion_mask to size advantages')
+
+ @staticmethod
+ def _validate_record(record: Any, sequence_length: int) -> dict[str, Any]:
+ if not isinstance(record, Mapping):
+ raise TypeError('turn_credit_invalid')
+ raw_pre_state = record.get('pre_state_key')
+ raw_post_state = record.get('post_state_key')
+ if raw_pre_state is None or raw_post_state is None:
+ raise ValueError('state_key_missing')
+ if not isinstance(raw_pre_state, str) or not isinstance(raw_post_state, str):
+ raise ValueError('state_key_invalid')
+ pre_state = raw_pre_state.strip()
+ post_state = raw_post_state.strip()
+ if not pre_state or not post_state:
+ raise ValueError('state_key_missing')
+ try:
+ start = int(record.get('action_token_start', -1))
+ end = int(record.get('action_token_end', -1))
+ except (TypeError, ValueError) as exc:
+ raise ValueError('action_span_invalid') from exc
+ if start < 0 or end <= start or end > sequence_length:
+ raise ValueError('action_span_invalid')
+ observation_start = record.get('observation_token_start')
+ observation_end = record.get('observation_token_end')
+ if (observation_start is None) != (observation_end is None):
+ raise ValueError('observation_span_invalid')
+ if observation_start is not None:
+ try:
+ observation_start = int(observation_start)
+ observation_end = int(observation_end)
+ except (TypeError, ValueError) as exc:
+ raise ValueError('observation_span_invalid') from exc
+ if (observation_start < end or observation_end <= observation_start or observation_end > sequence_length):
+ raise ValueError('observation_span_invalid')
+ policy_version = record.get('policy_version')
+ if policy_version is not None and (isinstance(policy_version, bool)
+ or not isinstance(policy_version, (int, str))):
+ raise ValueError('policy_version_invalid')
+ terminal_success = record.get('terminal_success')
+ if terminal_success is not None and not isinstance(terminal_success, bool):
+ raise ValueError('terminal_success_invalid')
+ parsed = dict(record)
+ parsed.update(
+ action_token_start=start,
+ action_token_end=end,
+ observation_token_start=observation_start,
+ observation_token_end=observation_end,
+ pre_state_key=pre_state,
+ post_state_key=post_state,
+ policy_version=policy_version,
+ terminal_success=terminal_success,
+ )
+ return parsed
+
+ def _distances(
+ self,
+ records: Sequence[tuple[int, Mapping[str, Any]]],
+ ) -> tuple[dict[str, int], int, int]:
+ reverse: dict[str, set[str]] = defaultdict(set)
+ nodes: set[str] = set()
+ edges: set[tuple[str, str]] = set()
+ success: set[str] = set()
+ for _, record in records:
+ pre_state, post_state = record['pre_state_key'], record['post_state_key']
+ nodes.update((pre_state, post_state))
+ edges.add((pre_state, post_state))
+ reverse[post_state].add(pre_state)
+ if record.get('terminal_success') is True:
+ success.add(post_state)
+ if not success:
+ raise ValueError('terminal_success_missing')
+ if len(nodes) > self.max_graph_nodes:
+ raise ValueError('graph_budget_exceeded')
+ if len(edges) > self.max_graph_edges:
+ raise ValueError('graph_budget_exceeded')
+ distances = {node: math.inf for node in nodes}
+ queue: deque[str] = deque()
+ for node in success:
+ distances[node] = 0
+ queue.append(node)
+ while queue:
+ child = queue.popleft()
+ for parent in reverse.get(child, ()):
+ candidate = distances[child] + 1
+ if candidate < distances[parent]:
+ distances[parent] = candidate
+ queue.append(parent)
+ return distances, len(nodes), len(edges)
+
+ def __call__(
+ self,
+ rewards: Sequence[float],
+ trajectories: Sequence[Mapping[str, Any]],
+ *,
+ num_generations: int,
+ sequence_lengths: Sequence[int] | None = None,
+ **kwargs,
+ ):
+ del kwargs
+ advantages, diagnostics = self.compute(
+ rewards,
+ trajectories,
+ num_generations=num_generations,
+ sequence_lengths=sequence_lengths,
+ )
+ self.last_diagnostics = diagnostics
+ return advantages
diff --git a/src/twinkle_agentic/envs/base.py b/src/twinkle_agentic/envs/base.py
index 215137513..0c6199cdb 100644
--- a/src/twinkle_agentic/envs/base.py
+++ b/src/twinkle_agentic/envs/base.py
@@ -159,6 +159,15 @@ def workspace(self) -> Optional[str]:
def reset(self, trajectory: Optional[Trajectory] = None) -> StepResult:
return StepResult()
+ def state_key(self) -> Optional[str]:
+ """Return an opaque key for the current task state, when supported.
+
+ The key must describe task state, not the transcript. Environments that
+ cannot provide a stable identifier keep the default ``None`` and are
+ excluded from state-conditioned credit assignment.
+ """
+ return None
+
@abstractmethod
def step(self, tool_name: str, arguments: Dict[str, Any]) -> StepResult:
raise NotImplementedError
diff --git a/src/twinkle_agentic/envs/env_tool.py b/src/twinkle_agentic/envs/env_tool.py
index 5732a75b7..d38a807ad 100644
--- a/src/twinkle_agentic/envs/env_tool.py
+++ b/src/twinkle_agentic/envs/env_tool.py
@@ -3,6 +3,7 @@
from typing import Any, Dict, List, Optional, Tuple
from twinkle.data_format.message import Tool as ToolInfo
+from twinkle_agentic.tools.base import ToolExecutionResult
from .base import Env, StepResult
@@ -21,16 +22,26 @@ def __init__(self,
self.last_result: Optional[StepResult] = None
def __call__(self, tool_name: str, arguments: Dict[str, Any]) -> str:
+ return self.call_result(tool_name, arguments).observation
+
+ def call_result(self, tool_name: str, arguments: Dict[str, Any]) -> ToolExecutionResult:
result = self._env.step(tool_name, arguments)
self.last_result = result
- return result.observation
+ return ToolExecutionResult(observation=result.observation, info=dict(result.info or {}))
def call_many(self, calls: List[Tuple[str, Dict[str, Any]]]) -> List[str]:
"""Batch through ``Env.step_batch``."""
+ return [result.observation for result in self.call_many_results(calls)]
+
+ def call_many_results(self, calls: List[Tuple[str, Dict[str, Any]]]) -> List[ToolExecutionResult]:
+ """Batch through ``Env.step_batch`` while preserving per-call info."""
results = self._env.step_batch(calls)
if results:
self.last_result = results[-1]
- return [r.observation for r in results]
+ return [ToolExecutionResult(observation=r.observation, info=dict(r.info or {})) for r in results]
+
+ def current_state_key(self) -> Optional[str]:
+ return self._env.state_key()
def tool_info(self) -> ToolInfo:
return {
diff --git a/src/twinkle_agentic/envs/openenv.py b/src/twinkle_agentic/envs/openenv.py
index a6a38b6d1..16e145e19 100644
--- a/src/twinkle_agentic/envs/openenv.py
+++ b/src/twinkle_agentic/envs/openenv.py
@@ -84,17 +84,20 @@ def _normalize_result(result) -> Dict[str, Any]:
'observation': _format_observation(result.get('observation', '')),
'reward': float(result.get('reward', 0.0)),
'done': bool(result.get('done', False)),
+ 'info': dict(result.get('info') or {}),
}
if hasattr(result, 'observation'):
return {
'observation': _format_observation(result.observation),
'reward': float(getattr(result, 'reward', 0.0) or 0.0),
'done': bool(getattr(result, 'done', False)),
+ 'info': dict(getattr(result, 'info', {}) or {}),
}
return {
'observation': _format_observation(result),
'reward': float(getattr(result, 'reward', 0.0) or 0.0),
'done': bool(getattr(result, 'done', False)),
+ 'info': {},
}
@@ -193,6 +196,7 @@ def reset(self, trajectory=None) -> StepResult:
observation=normalized['observation'],
reward=normalized['reward'],
done=normalized['done'],
+ info=normalized['info'],
)
def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult:
@@ -208,6 +212,7 @@ def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult:
observation=normalized['observation'],
reward=normalized['reward'],
done=normalized['done'],
+ info=normalized['info'],
)
def close(self) -> None:
@@ -434,10 +439,13 @@ def reset(self, trajectory=None) -> StepResult:
# session (and its WebSocket) is reused across episodes.
result = self._client.reset(**self.reset_kwargs)
self._last_result = result
+ info = dict(getattr(result, 'info', {}) or {})
+ info['episode_reward'] = self._episode_reward
return StepResult(
observation=_format_client_observation(getattr(result, 'observation', None)),
reward=float(getattr(result, 'reward', 0.0) or 0.0),
done=bool(getattr(result, 'done', False)),
+ info=info,
)
def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult:
@@ -458,11 +466,13 @@ def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult:
return StepResult(observation=f'Error: {e}', reward=0.0, done=False, info={'error': str(e)})
reward = float(getattr(result, 'reward', 0.0) or 0.0)
self._episode_reward += reward
+ info = dict(getattr(result, 'info', {}) or {})
+ info['episode_reward'] = self._episode_reward
return StepResult(
observation=_format_client_observation(getattr(result, 'observation', None)),
reward=reward,
done=bool(getattr(result, 'done', False)),
- info={'episode_reward': self._episode_reward},
+ info=info,
)
def tools(self) -> List[ToolInfo]:
diff --git a/src/twinkle_agentic/rollout/__init__.py b/src/twinkle_agentic/rollout/__init__.py
index 20e22288e..18d13f288 100644
--- a/src/twinkle_agentic/rollout/__init__.py
+++ b/src/twinkle_agentic/rollout/__init__.py
@@ -1,6 +1,7 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
from .base import Rollout
+from .credit import ToolEvent, TurnActionSpan, TurnCreditRecord
from .external import ExternalRollout
from .multi_turn import MultiTurnRollout
-__all__ = ['ExternalRollout', 'MultiTurnRollout', 'Rollout']
+__all__ = ['ExternalRollout', 'MultiTurnRollout', 'Rollout', 'ToolEvent', 'TurnActionSpan', 'TurnCreditRecord']
diff --git a/src/twinkle_agentic/rollout/credit.py b/src/twinkle_agentic/rollout/credit.py
new file mode 100644
index 000000000..ede9cd0d0
--- /dev/null
+++ b/src/twinkle_agentic/rollout/credit.py
@@ -0,0 +1,184 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""Optional state and tool provenance for multi-turn credit assignment."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Iterable, Mapping, Optional, Sequence, Tuple
+
+STATE_KEY_INFO = 'twinkle.state_key'
+TERMINAL_SUCCESS_INFO = 'twinkle.terminal_success'
+TOOL_EVENT_ID_INFO = 'twinkle.tool_event_id'
+MAX_STATE_KEY_CHARS = 256
+
+__all__ = [
+ 'ObservationSpan',
+ 'STATE_KEY_INFO',
+ 'TERMINAL_SUCCESS_INFO',
+ 'TOOL_EVENT_ID_INFO',
+ 'ToolEvent',
+ 'TurnActionSpan',
+ 'TurnCreditRecord',
+ 'TurnCreditState',
+ 'make_tool_events',
+ 'normalize_state_key',
+ 'qualified_state_key',
+ 'state_from_tool_infos',
+]
+
+
+@dataclass(frozen=True)
+class TurnActionSpan:
+ """Trainable action positions in the final labels index space."""
+
+ start: int
+ end: int
+
+
+@dataclass(frozen=True)
+class ObservationSpan:
+ """Masked positions appended between two generation turns."""
+
+ start: int
+ end: int
+
+
+@dataclass(frozen=True)
+class ToolEvent:
+ tool_call_id: str
+ tool_name: str
+ arguments_digest: str
+ result_digest: str
+ info: Mapping[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True)
+class TurnCreditRecord:
+ turn_id: int
+ action_token_start: int
+ action_token_end: int
+ observation_token_start: int | None
+ observation_token_end: int | None
+ pre_state_key: str
+ post_state_key: str
+ tool_events: tuple[ToolEvent, ...]
+ policy_version: int | str | None
+ terminal_success: bool | None
+
+ def to_dict(self) -> dict[str, Any]:
+ return asdict(self)
+
+
+@dataclass(frozen=True)
+class TurnCreditState:
+ available: bool
+ fallback_reason: str | None = None
+
+
+def normalize_state_key(value: Any) -> str | None:
+ """Validate one environment-provided state key."""
+ if value is None:
+ return None
+ if not isinstance(value, str):
+ raise TypeError(f'state_key must be a string or None, got {type(value).__name__}')
+ key = value.strip()
+ if not key:
+ raise ValueError('state_key must not be empty')
+ if len(key) > MAX_STATE_KEY_CHARS:
+ raise ValueError(f'state_key exceeds {MAX_STATE_KEY_CHARS} characters')
+ if any(ord(char) < 32 for char in key):
+ raise ValueError('state_key must not contain control characters')
+ return key
+
+
+def qualified_state_key(
+ environment_id: str | None,
+ task_id: str | None,
+ local_state_key: Any,
+) -> str | None:
+ """Build the rollout-scoped key expected by the state graph."""
+ if not environment_id or not task_id:
+ return None
+ local = normalize_state_key(local_state_key)
+ if local is None:
+ return None
+ return f'{environment_id}:{task_id}:{local}'
+
+
+def _digest(value: Any) -> str:
+ payload = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
+ return hashlib.sha256(payload.encode('utf-8')).hexdigest()
+
+
+def _tool_call_parts(tool_call: Any, index: int) -> tuple[str, str, Any, str]:
+ call = tool_call if isinstance(tool_call, Mapping) else {}
+ call_id = str(call.get('id') or call.get('tool_call_id') or f'tool-{index}')
+ function = call.get('function') if isinstance(call.get('function'), Mapping) else {}
+ name = str(function.get('name') or '')
+ arguments = function.get('arguments', {})
+ if isinstance(arguments, str):
+ try:
+ decoded = json.loads(arguments) if arguments.strip() else {}
+ except json.JSONDecodeError:
+ decoded = arguments
+ else:
+ decoded = arguments
+ return call_id, name, decoded, _digest(decoded)
+
+
+def make_tool_events(
+ tool_calls: Sequence[Any],
+ results: Sequence[Any],
+ *,
+ turn_id: int,
+) -> tuple[ToolEvent, ...]:
+ """Pair tool calls with observations or named error results."""
+ if len(tool_calls) != len(results):
+ raise ValueError('tool_calls and results must have equal length')
+ events = []
+ for index, (call, result) in enumerate(zip(tool_calls, results)):
+ call_id, name, _, arguments_digest = _tool_call_parts(call, index)
+ if isinstance(result, Mapping):
+ observation = result.get('observation', '')
+ info = dict(result.get('info') or {})
+ else:
+ observation = getattr(result, 'observation', result)
+ info = dict(getattr(result, 'info', {}) or {})
+ info.setdefault(TOOL_EVENT_ID_INFO, f'turn-{turn_id}/call-{call_id}')
+ events.append(
+ ToolEvent(
+ tool_call_id=call_id,
+ tool_name=name,
+ arguments_digest=arguments_digest,
+ result_digest=_digest(str(observation)),
+ info=info,
+ ))
+ return tuple(events)
+
+
+def state_from_tool_infos(infos: Iterable[Mapping[str, Any]], ) -> tuple[str | None, bool | None, str | None]:
+ """Return one state and terminal flag, or a conflict reason."""
+ state_keys = []
+ successes = []
+ for info in infos:
+ raw_state = info.get(STATE_KEY_INFO) if isinstance(info, Mapping) else None
+ if raw_state is not None:
+ state_keys.append(normalize_state_key(raw_state))
+ raw_success = info.get(TERMINAL_SUCCESS_INFO) if isinstance(info, Mapping) else None
+ if raw_success is not None:
+ if not isinstance(raw_success, bool):
+ raise TypeError(f'{TERMINAL_SUCCESS_INFO} must be bool when present')
+ successes.append(raw_success)
+ unique_states = set(state_keys)
+ unique_successes = set(successes)
+ if len(unique_states) > 1:
+ return None, None, 'state_conflict'
+ if len(unique_successes) > 1:
+ return None, None, 'terminal_success_conflict'
+ return (
+ next(iter(unique_states)) if unique_states else None,
+ next(iter(unique_successes)) if unique_successes else None,
+ None,
+ )
diff --git a/src/twinkle_agentic/rollout/ledger.py b/src/twinkle_agentic/rollout/ledger.py
index 053cd7be9..678c13951 100644
--- a/src/twinkle_agentic/rollout/ledger.py
+++ b/src/twinkle_agentic/rollout/ledger.py
@@ -32,6 +32,7 @@
from twinkle.data_format.sampling import SampledSequence
from twinkle.template.base import Template
from twinkle_agentic.utils.token_utils import _to_plain, append_ids, extend_with_bridge
+from .credit import ObservationSpan, ToolEvent, TurnActionSpan, TurnCreditRecord, TurnCreditState
class TurnLedger:
@@ -55,7 +56,16 @@ class TurnLedger:
which rebuilt them.
"""
- def __init__(self, template: Template, *, label: str = 'trajectory', max_tokens: Optional[int] = None) -> None:
+ def __init__(
+ self,
+ template: Template,
+ *,
+ label: str = 'trajectory',
+ max_tokens: Optional[int] = None,
+ track_turn_credits: bool = False,
+ initial_state_key: Optional[str] = None,
+ policy_version: Optional[int | str] = None,
+ ) -> None:
"""
Args:
template: encodes the opening turn and every bridge after it. One
@@ -70,9 +80,22 @@ def __init__(self, template: Template, *, label: str = 'trajectory', max_tokens:
self.template = template
self.label = label
self.max_tokens = max_tokens
+ if policy_version is not None and (isinstance(policy_version, bool)
+ or not isinstance(policy_version, (int, str))):
+ raise TypeError('policy_version must be an int, string, or None')
self._pif: Dict[str, Any] = {}
self._logprobs: List[Any] = []
self._turns = 0
+ self._track_turn_credits = track_turn_credits
+ self._initial_state_key = initial_state_key
+ self._current_state_key = initial_state_key
+ self._policy_version = policy_version
+ self._turn_credits: List[TurnCreditRecord] = []
+ self._turn_credit_state = self._initial_credit_state()
+ self._pending_turn_id: Optional[int] = None
+ self._pending_pre_state_key: Optional[str] = None
+ self._pending_action_span: Optional[TurnActionSpan] = None
+ self._last_observation_span: Optional[ObservationSpan] = None
# ---------------------------------------------------------------- reading
@@ -95,12 +118,29 @@ def turns(self) -> int:
"""Generations banked. Not turns *attempted*: a failed call is not here."""
return self._turns
+ @property
+ def turn_credits(self) -> List[TurnCreditRecord]:
+ return list(self._turn_credits)
+
+ @property
+ def turn_credit_state(self) -> TurnCreditState:
+ return self._turn_credit_state
+
def full(self) -> bool:
"""Has the episode reached :attr:`max_tokens`? Never true without one."""
if self.max_tokens is None:
return False
return len(self._pif.get('input_ids') or []) >= self.max_tokens
+ def _initial_credit_state(self) -> TurnCreditState:
+ if not self._track_turn_credits:
+ return TurnCreditState(available=False)
+ if self._initial_state_key is None:
+ return TurnCreditState(available=False, fallback_reason='state_key_missing')
+ if self._policy_version is None:
+ return TurnCreditState(available=False, fallback_reason='policy_version_missing')
+ return TurnCreditState(available=True)
+
# ---------------------------------------------------------------- writing
def open(self, trajectory: Trajectory, *, tools: Optional[List[Dict[str, Any]]] = None) -> None:
@@ -123,8 +163,15 @@ def open(self, trajectory: Trajectory, *, tools: Optional[List[Dict[str, Any]]]
self._pif = pif
self._logprobs = []
self._turns = 0
-
- def record(self, seq: SampledSequence) -> None:
+ self._turn_credits = []
+ self._current_state_key = self._initial_state_key
+ self._turn_credit_state = self._initial_credit_state()
+ self._pending_turn_id = None
+ self._pending_pre_state_key = None
+ self._pending_action_span = None
+ self._last_observation_span = None
+
+ def record(self, seq: SampledSequence) -> Optional[TurnActionSpan]:
"""Bank one generation, taking its ids from the sampler.
``new_input_feature`` is the prompt plus what was just sampled, already
@@ -138,9 +185,37 @@ def record(self, seq: SampledSequence) -> None:
raise RuntimeError(f'sampler returned a SampledSequence without '
f'new_input_feature.input_ids for {self.label}; '
f'cannot continue multi-turn.')
+ old_ids = list(self._pif.get('input_ids') or [])
self._pif = _to_plain(dict(seq.new_input_feature))
self._turns += 1
self._extend_logprobs(seq)
+ if not self._track_turn_credits:
+ return None
+ return self._begin_credit_turn(seq, old_ids)
+
+ def _begin_credit_turn(self, seq: SampledSequence, old_ids: Sequence[Any]) -> Optional[TurnActionSpan]:
+ old_length = len(old_ids)
+ new_ids = list(self._pif.get('input_ids') or [])
+ new_length = len(self._pif.get('input_ids') or [])
+ labels = self._pif.get('labels') or []
+ completion_mask = self._pif.get('completion_mask')
+ start, end = old_length - 1, new_length - 1
+ valid = (
+ old_length > 0 and new_length - old_length == len(seq.tokens) and new_ids[:old_length] == list(old_ids)
+ and new_ids[old_length:] == list(seq.tokens) and 0 <= start < end <= len(labels)
+ and (completion_mask is None or
+ (len(completion_mask) == len(labels) and all(completion_mask[pos] for pos in range(start, end)))))
+ if not valid:
+ self._turn_credit_state = TurnCreditState(False, 'action_span_invalid')
+ return None
+ span = TurnActionSpan(start=start, end=end)
+ self._pending_turn_id = self._turns
+ self._pending_pre_state_key = self._current_state_key
+ self._pending_action_span = span
+ self._last_observation_span = None
+ if self._pending_pre_state_key is None:
+ self._turn_credit_state = TurnCreditState(False, 'state_key_missing')
+ return span
def _extend_logprobs(self, seq: SampledSequence) -> None:
"""Append one generation's logprobs -- one per sampled token, or raise.
@@ -164,12 +239,60 @@ def observe(self, messages: Sequence[Dict[str, Any]]) -> bool:
words, and training on them teaches the model to predict its own
observations. False means the append did not fit.
"""
+ before_length = len(self._pif.get('input_ids') or [])
extended = extend_with_bridge(self._pif, list(messages), self.template)
if extended is None:
return False
self._pif = extended
+ after_length = len(self._pif.get('input_ids') or [])
+ if after_length > before_length:
+ self._last_observation_span = ObservationSpan(start=max(0, before_length - 1), end=after_length - 1)
return True
+ def complete_credit_turn(
+ self,
+ *,
+ post_state_key: Optional[str],
+ tool_events: Sequence[ToolEvent],
+ terminal_success: Optional[bool],
+ ) -> None:
+ if not self._track_turn_credits or self._pending_action_span is None:
+ return
+ if not self._turn_credit_state.available:
+ self.abort_credit_turn()
+ return
+ if post_state_key is None:
+ self._turn_credit_state = TurnCreditState(False, 'state_key_missing')
+ self.abort_credit_turn()
+ return
+ observation = self._last_observation_span
+ self._turn_credits.append(
+ TurnCreditRecord(
+ turn_id=self._pending_turn_id or self._turns,
+ action_token_start=self._pending_action_span.start,
+ action_token_end=self._pending_action_span.end,
+ observation_token_start=None if observation is None else observation.start,
+ observation_token_end=None if observation is None else observation.end,
+ pre_state_key=self._pending_pre_state_key or '',
+ post_state_key=post_state_key,
+ tool_events=tuple(tool_events),
+ policy_version=self._policy_version,
+ terminal_success=terminal_success,
+ ))
+ self._current_state_key = post_state_key
+ self._pending_turn_id = None
+ self._pending_pre_state_key = None
+ self._pending_action_span = None
+ self._last_observation_span = None
+
+ def abort_credit_turn(self, reason: Optional[str] = None) -> None:
+ if reason and self._turn_credit_state.available:
+ self._turn_credit_state = TurnCreditState(False, reason)
+ self._pending_turn_id = None
+ self._pending_pre_state_key = None
+ self._pending_action_span = None
+ self._last_observation_span = None
+
def adopt(self, input_feature: Dict[str, Any]) -> None:
"""Take an already-encoded feature as the current state.
@@ -245,6 +368,12 @@ def merge(self, trajectory: Trajectory, **fields: Any) -> Trajectory:
out.update(self._pif)
out['messages'] = list(self._pif.get('messages') or trajectory.get('messages') or [])
out['logprobs'] = self._logprobs if self._logprobs else None
+ if self._track_turn_credits:
+ out['turn_credits'] = [record.to_dict() for record in self._turn_credits]
+ out['turn_credit_status'] = {
+ 'available': self._turn_credit_state.available,
+ 'fallback_reason': self._turn_credit_state.fallback_reason,
+ }
out.update(fields)
return out
diff --git a/src/twinkle_agentic/rollout/multi_turn.py b/src/twinkle_agentic/rollout/multi_turn.py
index 87c10988a..b90b482d1 100644
--- a/src/twinkle_agentic/rollout/multi_turn.py
+++ b/src/twinkle_agentic/rollout/multi_turn.py
@@ -9,8 +9,10 @@
from twinkle.template.base import Template
from twinkle_agentic.protocol.api_sampler import APIGenerationError, APISampler
from twinkle_agentic.protocol.base import API
+from twinkle_agentic.tools.base import ToolExecutionResult
from twinkle_agentic.tools.tool_manager import ToolManager
from .base import MAX_FOLLOWUPS, STOP_GENERATION_ERROR, Rollout
+from .credit import make_tool_events, qualified_state_key, state_from_tool_infos
from .ledger import TurnLedger
from .trace import TraceWriter
@@ -179,6 +181,7 @@ def __init__(
use_base_model: bool = False,
stop_after_stuck_turns: int = 0,
max_malformed_retries: int = 2,
+ track_turn_credits: bool = False,
followup_fn: Optional[Callable[[Trajectory, int], Any]] = None,
api: Optional[API] = None,
response_callback: Optional[ResponseCallback] = None,
@@ -279,6 +282,7 @@ def __init__(
raise ValueError(f'max_malformed_retries must be >= 0, got '
f'{max_malformed_retries}')
self.max_malformed_retries = max_malformed_retries
+ self.track_turn_credits = track_turn_credits
# Called with (trajectory, how many follow-ups it has had already) at the
# moment an episode would end: because the model stopped calling tools,
# because it used up ``max_turns``, or because it was stopped for being
@@ -335,6 +339,10 @@ def _resolve_call(self, kwargs: Dict[str, Any], n: int) -> Dict[str, Any]:
'tool_managers': self._broadcast(kwargs.get('tool_manager', self.tool_manager), n, name='tool_manager'),
'followup_fn': kwargs.get('followup_fn', self.followup_fn),
'harnesses': self._resolve_harness(kwargs.get('harness', self.harness), n),
+ 'track_turn_credits': bool(kwargs.get('track_turn_credits', self.track_turn_credits)),
+ 'environment_ids': self._broadcast(kwargs.get('environment_id'), n, name='environment_id'),
+ 'task_ids': self._broadcast(kwargs.get('task_id'), n, name='task_id'),
+ 'policy_versions': self._broadcast(kwargs.get('policy_version'), n, name='policy_version'),
}
def _resolve_harness(self, harness, n: int) -> List[Any]:
@@ -362,11 +370,30 @@ def _run_episode(self, trajectory: Trajectory, index: int, ctx: Dict[str, Any],
followup_fn = ctx['followup_fn']
adapter_kwargs: Dict[str, Any] = ctx['adapter_kwargs']
response_callback: ResponseCallback = ctx['response_callback']
+ track_turn_credits = ctx['track_turn_credits']
+ requested_environment_id = ctx['environment_ids'][index]
+ requested_task_id = ctx['task_ids'][index]
+ requested_policy_version = ctx['policy_versions'][index]
+ environment_id = (
+ requested_environment_id if requested_environment_id is not None else trajectory.get('environment_id'))
+ task_id = requested_task_id if requested_task_id is not None else trajectory.get('task_id')
+ policy_version = (
+ requested_policy_version if requested_policy_version is not None else trajectory.get('policy_version'))
# The token account for this episode. Every id the trajectory ends up
# trained on passes through it; what stays in this function is the policy
# that decides when to add one. See ``ledger.py``.
- ledger = TurnLedger(self.template, label=f'trajectory {index}', max_tokens=self.max_trajectory_tokens)
+ initial_state_key = (
+ qualified_state_key(environment_id, task_id, self._current_state_key(tool_manager))
+ if track_turn_credits else None)
+ ledger = TurnLedger(
+ self.template,
+ label=f'trajectory {index}',
+ max_tokens=self.max_trajectory_tokens,
+ track_turn_credits=track_turn_credits,
+ initial_state_key=initial_state_key,
+ policy_version=policy_version,
+ )
# A trajectory that named no tools advertises the manager's, so the prompt
# lists what can actually be dispatched.
# A harness may shape the opening (system prompt, tool schema) before
@@ -528,6 +555,7 @@ def grant_followup() -> bool:
parse_errors = ([] if followups else self.template.tool_call_errors(seq.decoded or ''))
if parse_errors and malformed_turns < self.max_malformed_retries:
malformed_turns += 1
+ ledger.abort_credit_turn()
if not ledger.observe([_malformed_tool_message(parse_errors)]):
truncated = True
break
@@ -537,6 +565,7 @@ def grant_followup() -> bool:
# why this is not a second rollout.
if grant_followup():
continue
+ ledger.abort_credit_turn()
break
if ledger.turns >= self.max_turns:
@@ -554,7 +583,30 @@ def grant_followup() -> bool:
# messages the model did not write.
if tool_manager is None:
raise ValueError('the model emitted tool_calls but this trajectory has no ToolManager')
- observations = self._run_tools(tool_manager, tool_calls)
+ tool_results = self._run_tools_results(tool_manager, tool_calls)
+ observations = [result.observation for result in tool_results]
+ terminal_success = None
+ state_error = None
+ post_state_key = None
+ if track_turn_credits:
+ try:
+ local_post_state, terminal_success, state_error = state_from_tool_infos(
+ [result.info for result in tool_results])
+ except (TypeError, ValueError):
+ local_post_state, terminal_success, state_error = None, None, 'state_key_invalid'
+ current_state = self._current_state_key(tool_manager)
+ explicit_post_state = qualified_state_key(environment_id, task_id, local_post_state)
+ current_post_state = qualified_state_key(environment_id, task_id, current_state)
+ if explicit_post_state is not None and current_post_state is not None:
+ post_state_key = explicit_post_state
+ if explicit_post_state != current_post_state:
+ state_error = 'state_conflict'
+ else:
+ post_state_key = explicit_post_state or current_post_state
+ if state_error is not None:
+ ledger.abort_credit_turn(state_error)
+ elif post_state_key is None:
+ ledger.abort_credit_turn('state_key_missing')
if self.stop_after_stuck_turns:
keys = [_call_key(tc) for tc in tool_calls]
all_repeats = bool(keys) and all(k in seen_calls for k in keys)
@@ -570,7 +622,19 @@ def grant_followup() -> bool:
else:
tool_messages = _default_tool_messages(tool_calls, observations)
overflowed = not ledger.observe(tool_messages)
+ if track_turn_credits and not overflowed and state_error is None and post_state_key is not None:
+ try:
+ tool_events = make_tool_events(tool_calls, tool_results, turn_id=ledger.turns)
+ except (TypeError, ValueError):
+ ledger.abort_credit_turn('tool_event_invalid')
+ else:
+ ledger.complete_credit_turn(
+ post_state_key=post_state_key,
+ tool_events=tool_events,
+ terminal_success=terminal_success,
+ )
if overflowed:
+ ledger.abort_credit_turn('observation_overflow')
# Trajectory exceeded max_length.
truncated = True
else:
@@ -640,6 +704,29 @@ def _harness_tool_messages(harness, pif: Dict[str, Any], observations, tool_call
tail = shaped_msgs[len(prior):]
return tail or _default_tool_messages(tool_calls, observations)
+ @staticmethod
+ def _current_state_key(tool_manager: Optional[ToolManager]) -> Optional[str]:
+ return None if tool_manager is None else tool_manager.current_state_key()
+
+ @staticmethod
+ def _run_tools_results(
+ tool_manager: ToolManager,
+ tool_calls: List[Dict[str, Any]],
+ ) -> List[ToolExecutionResult]:
+ if hasattr(tool_manager, 'call_many_results'):
+ results = list(tool_manager.call_many_results(tool_calls))
+ else:
+ contents = tool_manager.call_many(tool_calls) if hasattr(
+ tool_manager, 'call_many') else [tool_manager(tool_call) for tool_call in tool_calls]
+ results = [ToolExecutionResult(observation='' if content is None else str(content)) for content in contents]
+ if len(results) < len(tool_calls):
+ failure = ToolExecutionResult(
+ observation=f'the tool manager returned {len(results)} results for {len(tool_calls)} calls',
+ info={'error': 'tool_result_count_mismatch'},
+ )
+ results.extend([failure] * (len(tool_calls) - len(results)))
+ return results[:len(tool_calls)]
+
@staticmethod
def _run_tools(tool_manager: ToolManager, tool_calls: List[Dict[str, Any]]) -> List[str]:
"""Run one turn's calls, through ``call_many`` when the manager has it.
@@ -652,11 +739,4 @@ def _run_tools(tool_manager: ToolManager, tool_calls: List[Dict[str, Any]]) -> L
A manager that answers with fewer results than calls leaves the rest
empty rather than shifting them onto the wrong call.
"""
- if hasattr(tool_manager, 'call_many'):
- contents = tool_manager.call_many(tool_calls)
- else:
- contents = [tool_manager(tc) for tc in tool_calls]
- obs = [''] * len(tool_calls)
- for i, content in enumerate(contents[:len(tool_calls)]):
- obs[i] = '' if content is None else str(content)
- return obs
+ return [result.observation for result in MultiTurnRollout._run_tools_results(tool_manager, tool_calls)]
diff --git a/src/twinkle_agentic/tools/base.py b/src/twinkle_agentic/tools/base.py
index 92496a695..1fa4a5769 100644
--- a/src/twinkle_agentic/tools/base.py
+++ b/src/twinkle_agentic/tools/base.py
@@ -1,10 +1,19 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
from typing import Any, Dict
from twinkle.data_format import Tool as ToolInfo
+@dataclass(frozen=True)
+class ToolExecutionResult:
+ """One tool call's observation and structured provenance."""
+
+ observation: str = ''
+ info: Dict[str, Any] = field(default_factory=dict)
+
+
class Tool(ABC):
@abstractmethod
diff --git a/src/twinkle_agentic/tools/tool_manager.py b/src/twinkle_agentic/tools/tool_manager.py
index 2bdf01410..fea373af6 100644
--- a/src/twinkle_agentic/tools/tool_manager.py
+++ b/src/twinkle_agentic/tools/tool_manager.py
@@ -6,7 +6,7 @@
from twinkle.data_format import ToolCall
from twinkle.data_format.message import Tool as ToolInfo
from twinkle.utils import get_logger
-from twinkle_agentic.tools.base import Tool
+from twinkle_agentic.tools.base import Tool, ToolExecutionResult
logger = get_logger()
@@ -122,26 +122,41 @@ def tool_infos(self) -> List[ToolInfo]:
return [t.tool_info() for t in self._tools.values()]
def __call__(self, tool_call: Union[ToolCall, Dict[str, Any]]) -> str:
+ return self.call_result(tool_call).observation
+
+ def call_result(self, tool_call: Union[ToolCall, Dict[str, Any]]) -> ToolExecutionResult:
name, args, err = _unpack_tool_call(tool_call)
if err:
- return err
+ return ToolExecutionResult(observation=err, info={'error': err})
if (tool := self._tools.get(name)) is None:
available = ', '.join(sorted(self._tools)) or '(none)'
hint = ''
if (suggestion := _suggest(name, self._tools)) is not None:
hint = (f' Did you mean {suggestion!r}? Tool names must be given in '
f'full, including the part before "---".')
- return f'Error: unknown tool {name!r}.{hint} Available: {available}.'
+ error = f'Error: unknown tool {name!r}.{hint} Available: {available}.'
+ return ToolExecutionResult(observation=error, info={'error': error})
try:
- return str(tool(name, args))
+ if hasattr(tool, 'call_result'):
+ result = tool.call_result(name, args)
+ return ToolExecutionResult(observation=str(result.observation), info=dict(result.info or {}))
+ return ToolExecutionResult(observation=str(tool(name, args)))
except Exception as e: # noqa
- return f'Error: tool {name!r} raised {type(e).__name__}: {e}'
+ error = f'Error: tool {name!r} raised {type(e).__name__}: {e}'
+ return ToolExecutionResult(observation=error, info={'error': error})
def call_many(
self,
tool_calls: Iterable[Union[ToolCall, Dict[str, Any]]],
max_workers: Optional[int] = None,
) -> List[str]:
+ return [result.observation for result in self.call_many_results(tool_calls, max_workers=max_workers)]
+
+ def call_many_results(
+ self,
+ tool_calls: Iterable[Union[ToolCall, Dict[str, Any]]],
+ max_workers: Optional[int] = None,
+ ) -> List[ToolExecutionResult]:
"""Execute many tool calls, preserving input order.
``tool_calls`` are the OpenAI-shaped dicts produced by
@@ -169,18 +184,18 @@ def call_many(
if not calls:
return []
if len(calls) == 1:
- return [self(calls[0])]
+ return [self.call_result(calls[0])]
unpacked = [_unpack_tool_call(tc) for tc in calls]
env = self._shared_env()
if env is not None:
- out: List[Optional[str]] = [None] * len(calls)
+ out: List[Optional[ToolExecutionResult]] = [None] * len(calls)
batched: List[Tuple[int, str, Dict[str, Any]]] = []
for i, (name, args, err) in enumerate(unpacked):
if err is None and name in self._tools:
batched.append((i, name, args))
else:
- out[i] = self(calls[i])
+ out[i] = self.call_result(calls[i])
try:
results = env.step_batch([(name, args) for _i, name, args in batched])
except Exception as e: # noqa
@@ -194,8 +209,8 @@ def call_many(
f'{type(e).__name__}: {e}')
failure = f'Error: tool batch did not run: {type(e).__name__}: {e}'
for i, _name, _args in batched:
- out[i] = failure
- return ['' if x is None else x for x in out]
+ out[i] = ToolExecutionResult(observation=failure, info={'error': failure})
+ return [ToolExecutionResult() if x is None else x for x in out]
if len(results) != len(batched):
# Same reasoning: a short result list means the calls did not all
# run, and pairing them up by position would report one call's
@@ -205,19 +220,27 @@ def call_many(
failure = (f'Error: tool batch did not run: the environment returned '
f'{len(results)} results for {len(batched)} calls.')
for i, _name, _args in batched:
- out[i] = failure
- return ['' if x is None else x for x in out]
+ out[i] = ToolExecutionResult(observation=failure, info={'error': failure})
+ return [ToolExecutionResult() if x is None else x for x in out]
for (i, _name, _args), r in zip(batched, results):
- out[i] = r.observation if hasattr(r, 'observation') else str(r)
- return ['' if x is None else x for x in out]
+ observation = r.observation if hasattr(r, 'observation') else str(r)
+ info = dict(getattr(r, 'info', {}) or {})
+ out[i] = ToolExecutionResult(observation=observation, info=info)
+ return [ToolExecutionResult() if x is None else x for x in out]
workers = max_workers or min(32, len(calls))
- out = [None] * len(calls)
+ out: List[Optional[ToolExecutionResult]] = [None] * len(calls)
with ThreadPoolExecutor(max_workers=workers) as pool:
- futs = {pool.submit(self, tc): i for i, tc in enumerate(calls)}
+ futs = {pool.submit(self.call_result, tc): i for i, tc in enumerate(calls)}
for fut in as_completed(futs):
out[futs[fut]] = fut.result()
- return ['' if x is None else x for x in out]
+ return [ToolExecutionResult() if x is None else x for x in out]
+
+ def current_state_key(self) -> Optional[str]:
+ env = self._shared_env()
+ if env is None:
+ return None
+ return env.state_key()
def _shared_env(self):
"""Return the Env shared by every registered EnvTool, else None."""
diff --git a/tests/advantage/test_graphcredit_gate1.py b/tests/advantage/test_graphcredit_gate1.py
new file mode 100644
index 000000000..667486388
--- /dev/null
+++ b/tests/advantage/test_graphcredit_gate1.py
@@ -0,0 +1,142 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import importlib.util
+import sys
+from types import SimpleNamespace
+from pathlib import Path
+
+import torch
+import pytest
+
+from twinkle.advantage import StateGraphAdvantage
+
+
+MODULE_PATH = Path(__file__).resolve().parents[2] / 'cookbook/rl/graphcredit/gate1.py'
+SPEC = importlib.util.spec_from_file_location('graphcredit_gate1', MODULE_PATH)
+assert SPEC is not None and SPEC.loader is not None
+gate1 = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = gate1
+SPEC.loader.exec_module(gate1)
+
+
+def test_grid_task_and_parse_action():
+ task = gate1.make_task(7, grid_size=5, distance=2)
+ assert abs(task.start[0] - task.goal[0]) + abs(task.start[1] - task.goal[1]) == 2
+ assert gate1.parse_action(' RIGHT ') == 'right'
+ assert gate1.parse_action('right') == 'right'
+ assert gate1.parse_action('I choose a wall') is None
+
+
+def test_wrong_move_ends_episode_without_progress():
+ task = gate1.GridTask(
+ task_uid=1,
+ grid_size=5,
+ distance=2,
+ start=(2, 2),
+ goal=(2, 4),
+ )
+ episode = gate1.Episode(task=task, uid='bad', position=task.start)
+
+ gate1.step_episode(episode, 'left')
+
+ assert episode.done is True
+ assert episode.success is False
+ assert episode.reward == 0.0
+
+
+def _episode(task, actions):
+ episode = gate1.Episode(task=task, uid='episode', position=task.start)
+ for turn_id, action in enumerate(actions):
+ pre_state = task.state_key(episode.position)
+ start = 4 + turn_id * 4
+ gate1.step_episode(episode, action)
+ post_state = (
+ task.state_key(episode.position)
+ if episode.success or not episode.done else task.dead_state_key
+ )
+ episode.turns.append(
+ gate1.EpisodeTurn(
+ prompt='',
+ action=action,
+ pre_state_key=pre_state,
+ post_state_key=post_state,
+ action_token_start=start,
+ action_token_end=start + 2,
+ terminal_success=True if episode.success else False,
+ ))
+ if episode.done:
+ break
+ episode.input_ids = list(range(4 + len(episode.turns) * 4 + 2))
+ return episode
+
+
+def test_graph_credit_prefers_useful_intermediate_action():
+ task = gate1.GridTask(
+ task_uid=2,
+ grid_size=5,
+ distance=2,
+ start=(2, 2),
+ goal=(2, 4),
+ )
+ success = _episode(task, ['right', 'right'])
+ failure = _episode(task, ['left'])
+ trajectories = [
+ success.to_trajectory(policy_version=1),
+ failure.to_trajectory(policy_version=1),
+ ]
+
+ advantage = StateGraphAdvantage(beta_episode=0.0, beta_graph=1.0)
+ result = advantage([1.0, 0.0], trajectories, num_generations=2)
+
+ success_start = success.turns[0].action_token_start
+ failure_start = failure.turns[0].action_token_start
+ assert torch.all(result[0, success_start:success_start + 2] > 0)
+ assert torch.all(result[1, failure_start:failure_start + 2] < 0)
+
+
+def test_train_counts_completed_updates_but_not_skipped_attempts(monkeypatch):
+ task = gate1.GridTask(
+ task_uid=3,
+ grid_size=5,
+ distance=2,
+ start=(2, 2),
+ goal=(2, 4),
+ )
+ episode = _episode(task, ['right', 'right'])
+ sampled = (task, [episode])
+ samples = iter([None, sampled, None, sampled])
+ policy_versions = []
+
+ monkeypatch.setattr(gate1, 'evaluate', lambda *args, **kwargs: {
+ 'success_rate': 0.0,
+ 'invalid_rate': 0.0,
+ })
+ monkeypatch.setattr(gate1, 'reset_peak_hbm', lambda: None)
+ monkeypatch.setattr(gate1, 'synchronize', lambda device: None)
+ monkeypatch.setattr(gate1, 'peak_hbm_gb', lambda: 0.0)
+ monkeypatch.setattr(gate1, 'sample_training_group', lambda *args, **kwargs: next(samples))
+
+ def compute_advantages(mode, episodes, policy_version, group_size, args):
+ policy_versions.append(policy_version)
+ return torch.zeros(1), {'fallback_counts': {}}
+
+ monkeypatch.setattr(gate1, 'compute_advantages', compute_advantages)
+ monkeypatch.setattr(gate1, 'policy_gradient_update', lambda *args, **kwargs: (0.0, 1))
+ args = SimpleNamespace(
+ learning_rate=5e-5,
+ steps=2,
+ max_step_attempts=3,
+ eval_every=0,
+ seed=20261119,
+ mode='graphcredit',
+ group_size=2,
+ max_grad_norm=1.0,
+ )
+ tokenizer = SimpleNamespace(pad_token_id=0)
+
+ result = gate1.train(torch.nn.Linear(1, 1), tokenizer, args, torch.device('cpu'))
+
+ assert result['completed_steps'] == 2
+ assert result['attempted_steps'] == 4
+ assert result['skipped_steps'] == 2
+ assert policy_versions == [1, 2]
+ assert [record['step'] for record in result['history'] if not record.get('skipped')] == [1, 2]
diff --git a/tests/advantage/test_state_graph_advantage.py b/tests/advantage/test_state_graph_advantage.py
new file mode 100644
index 000000000..cc674715f
--- /dev/null
+++ b/tests/advantage/test_state_graph_advantage.py
@@ -0,0 +1,161 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import pytest
+import torch
+
+from twinkle.advantage import GRPOAdvantage, StateGraphAdvantage
+
+
+def _trajectory(pre_state, post_state, success, *, version=1, span=(1, 2)):
+ return {
+ 'labels': [0] * 6,
+ 'turn_credit_status': {
+ 'available': True,
+ 'fallback_reason': None,
+ },
+ 'turn_credits': [{
+ 'turn_id': 1,
+ 'action_token_start': span[0],
+ 'action_token_end': span[1],
+ 'pre_state_key': pre_state,
+ 'post_state_key': post_state,
+ 'policy_version': version,
+ 'terminal_success': success,
+ }],
+ }
+
+
+class TestStateGraphAdvantage:
+
+ def test_adds_state_relative_credit_to_action_tokens(self):
+ trajectories = [
+ _trajectory('s0', 'success', True),
+ _trajectory('s0', 'dead', None),
+ _trajectory('s1', 'success', True),
+ _trajectory('s1', 'dead', None),
+ ]
+ advantage = StateGraphAdvantage()
+ result = advantage([1.0, 0.0, 1.0, 0.0], trajectories, num_generations=4)
+
+ episode = GRPOAdvantage()(torch.tensor([1.0, 0.0, 1.0, 0.0]), num_generations=4)
+ assert torch.allclose(result[:, 0], episode)
+ assert torch.all(result[0, 1] > episode[0])
+ assert torch.all(result[1, 1] < episode[1])
+ assert advantage.last_diagnostics.comparable_state_group_count == 2
+ assert advantage.last_diagnostics.transition_count == 4
+ assert advantage.last_diagnostics.fallback_counts == {}
+
+ def test_missing_success_is_exact_grpo_fallback(self):
+ trajectories = [
+ _trajectory('s0', 'dead0', None),
+ _trajectory('s0', 'dead1', None),
+ _trajectory('s0', 'dead2', None),
+ _trajectory('s0', 'dead3', None),
+ ]
+ rewards = [0.0, 1.0, 0.0, 2.0]
+ advantage = StateGraphAdvantage()
+ result = advantage(rewards, trajectories, num_generations=4)
+ episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=4)
+
+ assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6))
+ assert advantage.last_diagnostics.fallback_counts['terminal_success_missing'] == 1
+
+ def test_policy_version_mismatch_is_exact_grpo_fallback(self):
+ trajectories = [
+ _trajectory('s0', 'success', True, version=1),
+ _trajectory('s0', 'dead', None, version=2),
+ ]
+ rewards = [1.0, 0.0]
+ advantage = StateGraphAdvantage()
+ result = advantage(rewards, trajectories, num_generations=2)
+ episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2)
+
+ assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6))
+ assert advantage.last_diagnostics.fallback_counts['policy_version_mismatch'] == 1
+
+ def test_partially_missing_policy_version_is_exact_grpo_fallback(self):
+ trajectories = [
+ _trajectory('s0', 'success', True, version=1),
+ _trajectory('s0', 'dead', None, version=None),
+ ]
+ rewards = [1.0, 0.0]
+ advantage = StateGraphAdvantage()
+ result = advantage(rewards, trajectories, num_generations=2)
+ episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2)
+
+ assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6))
+ assert advantage.last_diagnostics.fallback_counts['policy_version_missing'] == 1
+
+ def test_fully_missing_policy_version_is_exact_grpo_fallback(self):
+ trajectories = [
+ _trajectory('s0', 'success', True, version=None),
+ _trajectory('s0', 'dead', None, version=None),
+ ]
+ rewards = [1.0, 0.0]
+ advantage = StateGraphAdvantage()
+ result = advantage(rewards, trajectories, num_generations=2)
+ episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2)
+
+ assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6))
+ assert advantage.last_diagnostics.fallback_counts['policy_version_missing'] == 1
+
+ def test_policy_version_zero_is_available(self):
+ trajectories = [
+ _trajectory('s0', 'success', True, version=0),
+ _trajectory('s0', 'dead', None, version=0),
+ ]
+ advantage = StateGraphAdvantage()
+ result = advantage([1.0, 0.0], trajectories, num_generations=2)
+
+ assert result.shape == (2, 6)
+ assert advantage.last_diagnostics.fallback_counts == {}
+
+ @pytest.mark.parametrize('field,value,reason', [
+ ('pre_state_key', 1, 'state_key_invalid'),
+ ('policy_version', [], 'policy_version_invalid'),
+ ('terminal_success', 'yes', 'terminal_success_invalid'),
+ ('observation_token_start', 1, 'observation_span_invalid'),
+ ])
+ def test_invalid_contract_field_is_exact_grpo_fallback(self, field, value, reason):
+ trajectories = [_trajectory('s0', 'success', True), _trajectory('s0', 'dead', None)]
+ trajectories[0]['turn_credits'][0][field] = value
+ rewards = [1.0, 0.0]
+ advantage = StateGraphAdvantage()
+ result = advantage(rewards, trajectories, num_generations=2)
+ episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2)
+
+ assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6))
+ assert advantage.last_diagnostics.fallback_counts[reason] == 1
+
+ def test_insufficient_state_support_leaves_only_episode_credit(self):
+ trajectories = [
+ _trajectory('s0', 'success', True),
+ _trajectory('s1', 'dead', None),
+ ]
+ rewards = [1.0, 0.0]
+ advantage = StateGraphAdvantage()
+ result = advantage(rewards, trajectories, num_generations=2)
+ episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2)
+
+ assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6))
+ assert advantage.last_diagnostics.comparable_state_group_count == 0
+
+ def test_invalid_span_skips_that_row(self):
+ trajectories = [
+ _trajectory('s0', 'success', True, span=(0, 1)),
+ _trajectory('s0', 'dead', None, span=(7, 9)),
+ ]
+ rewards = [1.0, 0.0]
+ advantage = StateGraphAdvantage()
+ result = advantage(rewards, trajectories, num_generations=2)
+ episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2)
+
+ assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6))
+ assert advantage.last_diagnostics.fallback_counts['action_span_invalid'] == 1
+
+ def test_validation(self):
+ with pytest.raises(ValueError, match='at least two'):
+ StateGraphAdvantage()([1.0], [_trajectory('s', 't', True)], num_generations=1)
+ with pytest.raises(ValueError, match='complete groups'):
+ StateGraphAdvantage()([1.0], [_trajectory('s', 't', True)], num_generations=2)
+ with pytest.raises(ValueError, match='min_state_support'):
+ StateGraphAdvantage(min_state_support=1)
diff --git a/tests/twinkle_agentic/test_multi_turn_turn_credit.py b/tests/twinkle_agentic/test_multi_turn_turn_credit.py
new file mode 100644
index 000000000..94a47e7ec
--- /dev/null
+++ b/tests/twinkle_agentic/test_multi_turn_turn_credit.py
@@ -0,0 +1,110 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import json
+import sys
+import types
+
+try:
+ import resource # noqa: F401
+except ModuleNotFoundError:
+ stub = types.ModuleType('resource')
+ stub.RLIMIT_AS = 0
+ stub.setrlimit = lambda *_args, **_kwargs: None
+ sys.modules['resource'] = stub
+
+from twinkle_agentic.envs.base import Env, StepResult
+from twinkle_agentic.envs.env_tool import EnvTool
+from twinkle_agentic.rollout.credit import STATE_KEY_INFO, TERMINAL_SUCCESS_INFO
+from twinkle_agentic.rollout.multi_turn import MultiTurnRollout
+from twinkle_agentic.tools.tool_manager import ToolManager
+from tests.twinkle_agentic.test_multi_turn_rollout import FakeSampler, FakeTemplate, FakeTokenizer
+
+
+class StatefulEnv(Env):
+
+ def __init__(self):
+ self.state = 's0'
+
+ def state_key(self):
+ return self.state
+
+ def step(self, tool_name, arguments):
+ self.state = arguments.get('next_state', self.state)
+ return StepResult(
+ observation=f'state={self.state}',
+ info={
+ STATE_KEY_INFO: self.state,
+ TERMINAL_SUCCESS_INFO: arguments.get('success'),
+ },
+ )
+
+
+def _rollout(sampler, template, tool_manager, *, track):
+ return MultiTurnRollout(
+ sampler=sampler,
+ template=template,
+ tool_manager=tool_manager,
+ max_turns=3,
+ track_turn_credits=track,
+ )
+
+
+def test_multi_turn_emits_state_key_and_action_span():
+ tokenizer = FakeTokenizer()
+ template = FakeTemplate(tokenizer)
+ sampler = FakeSampler(template)
+ env = StatefulEnv()
+ tool_manager = ToolManager(EnvTool.from_schemas(env, [{
+ 'type': 'function',
+ 'function': {
+ 'name': 'act'
+ }
+ }]))
+ sampler.queue(
+ ''
+ + json.dumps({
+ 'name': 'act',
+ 'arguments': {
+ 'next_state': 's1',
+ 'success': True,
+ },
+ })
+ + '')
+ sampler.queue('done')
+ rollout = _rollout(sampler, template, tool_manager, track=True)
+ trajectory = rollout(
+ [{
+ 'messages': [{
+ 'role': 'user',
+ 'content': 'go'
+ }],
+ }],
+ environment_id='env',
+ task_id='task',
+ policy_version=7,
+ )[0]
+
+ assert trajectory['turn_credit_status']['available'] is True
+ assert len(trajectory['turn_credits']) == 1
+ credit = trajectory['turn_credits'][0]
+ assert credit['pre_state_key'] == 'env:task:s0'
+ assert credit['post_state_key'] == 'env:task:s1'
+ assert credit['action_token_start'] < credit['action_token_end']
+ assert credit['policy_version'] == 7
+ assert credit['terminal_success'] is True
+ assert credit['tool_events'][0]['tool_name'] == 'act'
+
+
+def test_multi_turn_keeps_turn_credit_fields_absent_by_default():
+ tokenizer = FakeTokenizer()
+ template = FakeTemplate(tokenizer)
+ sampler = FakeSampler(template)
+ sampler.queue('done')
+ trajectory = _rollout(sampler, template, None, track=False)(
+ [{
+ 'messages': [{
+ 'role': 'user',
+ 'content': 'go'
+ }],
+ }])[0]
+ assert 'turn_credits' not in trajectory
+ assert 'turn_credit_status' not in trajectory
diff --git a/tests/twinkle_agentic/test_turn_credit.py b/tests/twinkle_agentic/test_turn_credit.py
new file mode 100644
index 000000000..0b3a943be
--- /dev/null
+++ b/tests/twinkle_agentic/test_turn_credit.py
@@ -0,0 +1,219 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+import sys
+import types
+
+try:
+ import resource # noqa: F401
+except ModuleNotFoundError:
+ stub = types.ModuleType('resource')
+ stub.RLIMIT_AS = 0
+ stub.setrlimit = lambda *_args, **_kwargs: None
+ sys.modules['resource'] = stub
+
+from twinkle.data_format.sampling import SampledSequence
+from twinkle_agentic.envs.base import Env, StepResult
+from twinkle_agentic.envs.env_tool import EnvTool
+from twinkle_agentic.rollout.credit import (
+ STATE_KEY_INFO,
+ TERMINAL_SUCCESS_INFO,
+ make_tool_events,
+ qualified_state_key,
+ state_from_tool_infos,
+)
+from twinkle_agentic.rollout.ledger import TurnLedger
+from twinkle_agentic.tools.tool_manager import ToolManager
+
+
+class StatefulEnv(Env):
+
+ def __init__(self):
+ self.state = 's0'
+
+ def state_key(self):
+ return self.state
+
+ def step(self, tool_name, arguments):
+ return self.step_batch([(tool_name, arguments)])[0]
+
+ def step_batch(self, calls):
+ results = []
+ for _, arguments in calls:
+ self.state = arguments.get('next_state', self.state)
+ results.append(
+ StepResult(
+ observation=f'reached {self.state}',
+ info={
+ STATE_KEY_INFO: self.state,
+ TERMINAL_SUCCESS_INFO: arguments.get('success'),
+ },
+ ))
+ return results
+
+
+class FakeTemplate:
+
+ def encode(self, trajectory, add_generation_prompt=False):
+ del add_generation_prompt
+ messages = trajectory.get('messages') or []
+ ids = list(range(len(messages) + 2))
+ return {
+ 'messages': messages,
+ 'input_ids': ids,
+ 'labels': list(ids),
+ 'completion_mask': [0] * len(ids),
+ }
+
+
+def test_tool_manager_preserves_info_without_changing_call_many():
+ env = StatefulEnv()
+ manager = ToolManager(EnvTool.from_schemas(env, [{
+ 'type': 'function',
+ 'function': {
+ 'name': 'act'
+ }
+ }]))
+ calls = [{
+ 'id': 'c1',
+ 'type': 'function',
+ 'function': {
+ 'name': 'act',
+ 'arguments': {
+ 'next_state': 's1'
+ }
+ },
+ }]
+
+ rich = manager.call_many_results(calls)
+ assert len(rich) == 1
+ assert rich[0].observation == 'reached s1'
+ assert rich[0].info[STATE_KEY_INFO] == 's1'
+ assert manager.call_many(calls) == ['reached s1']
+ assert manager.current_state_key() == 's1'
+
+
+def test_tool_manager_marks_unknown_calls_without_losing_metadata():
+ manager = ToolManager()
+ result = manager.call_result({
+ 'id': 'missing',
+ 'function': {
+ 'name': 'not_registered',
+ 'arguments': {}
+ }
+ })
+ assert 'unknown tool' in result.observation
+ assert 'error' in result.info
+
+
+def test_state_and_tool_provenance_helpers():
+ assert qualified_state_key('env', 'task', ' s1 ') == 'env:task:s1'
+ assert qualified_state_key(None, 'task', 's1') is None
+ assert state_from_tool_infos([
+ {STATE_KEY_INFO: 's1'},
+ {STATE_KEY_INFO: 's1', TERMINAL_SUCCESS_INFO: True},
+ ]) == ('s1', True, None)
+ assert state_from_tool_infos([
+ {STATE_KEY_INFO: 's1'},
+ {STATE_KEY_INFO: 's2'},
+ ]) == (None, None, 'state_conflict')
+
+ events = make_tool_events(
+ [{
+ 'id': 'a',
+ 'function': {
+ 'name': 'act',
+ 'arguments': {'next_state': 's1'}
+ }
+ }],
+ [StepResult(observation='ok', info={STATE_KEY_INFO: 's1'})],
+ turn_id=1,
+ )
+ assert events[0].tool_call_id == 'a'
+ assert events[0].tool_name == 'act'
+ assert len(events[0].arguments_digest) == 64
+ assert len(events[0].result_digest) == 64
+
+
+def test_turn_ledger_records_action_span_and_state_edge():
+ ledger = TurnLedger(
+ FakeTemplate(),
+ track_turn_credits=True,
+ initial_state_key='env:task:s0',
+ policy_version=3,
+ )
+ ledger.open({'messages': [{'role': 'user', 'content': 'go'}]})
+ old_length = len(ledger.input_feature['input_ids'])
+ new_feature = {
+ **ledger.input_feature,
+ 'input_ids': [0, 1, 2, 10, 11, 12],
+ 'labels': [0, 1, 10, 11, 12, -100],
+ 'completion_mask': [0, 0, 1, 1, 1, 0],
+ }
+ span = ledger.record(SampledSequence(
+ stop_reason='stop',
+ tokens=[10, 11, 12],
+ logprobs=None,
+ new_input_feature=new_feature,
+ ))
+ assert span.start == old_length - 1
+ assert span.end == len(new_feature['input_ids']) - 1
+ ledger.complete_credit_turn(
+ post_state_key='env:task:s1',
+ tool_events=(),
+ terminal_success=True,
+ )
+ trajectory = ledger.merge({'messages': []})
+ assert trajectory['turn_credit_status']['available'] is True
+ record = trajectory['turn_credits'][0]
+ assert record['pre_state_key'] == 'env:task:s0'
+ assert record['post_state_key'] == 'env:task:s1'
+ assert record['action_token_start'] == span.start
+ assert record['action_token_end'] == span.end
+ assert record['policy_version'] == 3
+
+
+def test_turn_ledger_marks_missing_initial_state_as_fallback():
+ ledger = TurnLedger(FakeTemplate(), track_turn_credits=True)
+ trajectory = ledger.merge({'messages': []})
+ assert trajectory['turn_credit_status'] == {
+ 'available': False,
+ 'fallback_reason': 'state_key_missing',
+ }
+
+
+def test_turn_ledger_requires_policy_version_when_tracking():
+ ledger = TurnLedger(FakeTemplate(), track_turn_credits=True, initial_state_key='env:task:s0')
+ trajectory = ledger.merge({'messages': []})
+ assert trajectory['turn_credit_status'] == {
+ 'available': False,
+ 'fallback_reason': 'policy_version_missing',
+ }
+
+
+def test_turn_ledger_rejects_non_sampled_tail():
+ ledger = TurnLedger(
+ FakeTemplate(),
+ track_turn_credits=True,
+ initial_state_key='env:task:s0',
+ policy_version=0,
+ )
+ ledger.open({'messages': [{'role': 'user', 'content': 'go'}]})
+ old_feature = ledger.input_feature
+ new_feature = {
+ **old_feature,
+ 'input_ids': [0, 1, 2, 10, 11, 99],
+ 'labels': [0, 1, 10, 11, 99, -100],
+ 'completion_mask': [0, 0, 1, 1, 1, 0],
+ }
+ span = ledger.record(SampledSequence(
+ stop_reason='stop',
+ tokens=[10, 11, 12],
+ logprobs=None,
+ new_input_feature=new_feature,
+ ))
+ trajectory = ledger.merge({'messages': []})
+
+ assert span is None
+ assert trajectory['turn_credit_status'] == {
+ 'available': False,
+ 'fallback_reason': 'action_span_invalid',
+ }