diff --git a/cookbook/rl/graphcredit/README.md b/cookbook/rl/graphcredit/README.md new file mode 100644 index 000000000..8922ffecf --- /dev/null +++ b/cookbook/rl/graphcredit/README.md @@ -0,0 +1,20 @@ +# GraphCredit Gate 0 + +该目录只包含离线 Gate 0 探针,不进入 Twinkle 核心训练路径。 + +Gate 0 使用确定性状态机生成多轮轨迹,验证以下前提: + +- 环境显式状态键覆盖率; +- 同一 prompt 下同状态动作的支撑度; +- 状态图 credit 对正确动作和错误动作的排序能力; +- 无成功终态时 graph advantage 是否严格为零; +- 纯 CPU 图构造和 advantage 计算耗时; +- 是否产生额外 policy forward。 + +运行: + +```bash +PYTHONPATH=src python cookbook/rl/graphcredit/gate0.py +``` + +Gate 0 未通过时,不进入 `TurnCredit` 或 state-graph advantage 的核心实现。 diff --git a/cookbook/rl/graphcredit/README_GATE1.md b/cookbook/rl/graphcredit/README_GATE1.md new file mode 100644 index 000000000..0a3101def --- /dev/null +++ b/cookbook/rl/graphcredit/README_GATE1.md @@ -0,0 +1,69 @@ +# GraphCredit Gate 1 + +该实验在保持 rollout、组大小、模型、LoRA 配置、更新步数和评测任务完全一致的条件下, +比较 episode-level GRPO 与 `StateGraphAdvantage`。 + +## 任务 + +二维网格只要求连续两步靠近目标。错误方向会立即终止 episode。episode reward 只使用 +最终成功或失败: + +- 成功:`1.0` +- 失败:`0.0` + +因此 GRPO 只能把最终结果广播给 episode 内所有动作,而 GraphCredit 可以根据 +`pre_state` 分组,给“当前动作正确但后续失败”的中间动作更高信用。 + +## A2 运行 + +```bash +source /usr/local/Ascend/ascend-toolkit/set_env.sh +export PYTHONPATH="$PWD/src" + +python cookbook/rl/graphcredit/gate1.py \ + --model-path /workspace/shared_assets/models/Qwen/Qwen3-0.6B \ + --mode grpo \ + --steps 20 \ + --eval-tasks 24 \ + --group-size 8 \ + --device npu:0 \ + --output-dir outputs/graphcredit-gate1/grpo + +python cookbook/rl/graphcredit/gate1.py \ + --model-path /workspace/shared_assets/models/Qwen/Qwen3-0.6B \ + --mode graphcredit \ + --steps 20 \ + --eval-tasks 24 \ + --group-size 8 \ + --device npu:0 \ + --output-dir outputs/graphcredit-gate1/graphcredit +``` + +两次运行必须使用相同 `--seed`、模型、任务范围、group size 和训练步数。脚本输出: + +- `adapter/`:训练后的 LoRA; +- `result.json`:逐步 reward、loss、耗时、峰值 HBM 和 GraphCredit diagnostics; +- 标准输出:每一步一行 JSON,便于远端日志采集。 + +也可以使用统一启动脚本: + +```bash +bash cookbook/rl/graphcredit/run_gate1_a2.sh grpo 20 24 +bash cookbook/rl/graphcredit/run_gate1_a2.sh graphcredit 20 24 +``` + +可通过环境变量覆盖难度和周期性评测,例如: + +```bash +DISTANCE=3 GRID_SIZE=7 EVAL_EVERY=4 \ + bash cookbook/rl/graphcredit/run_gate1_a2.sh graphcredit 12 16 +``` + +`--steps` 表示成功优化更新次数,不是跳过组也计数的尝试次数。候选训练组最多允许 +`MAX_STEP_ATTEMPTS` 倍尝试,避免两臂实际更新预算不同。 + +## 证据边界 + +该实验验证 advantage consumer 的训练方向,不替代 Twinkle 完整 `GRPOLoss` 和生产训练 +入口。它没有 reference KL、多 epoch PPO 或异步 rollout。只有在 held-out success +稳定优于同预算 GRPO 后,才进入真实 Agentic 任务和多种子验证。 diff --git a/cookbook/rl/graphcredit/gate0.py b/cookbook/rl/graphcredit/gate0.py new file mode 100644 index 000000000..70f43424c --- /dev/null +++ b/cookbook/rl/graphcredit/gate0.py @@ -0,0 +1,347 @@ +"""Offline Gate 0 probe for the proposed TurnCredit state graph. + +This probe deliberately does not import or modify Twinkle's rollout path. It +tests the proposed state-key and graph semantics on deterministic trajectories +before any core implementation is allowed. +""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +import time +from collections import defaultdict, deque +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterable + + +@dataclass(frozen=True) +class Transition: + prompt_id: str + rollout_id: str + turn_id: int + pre_state: str + action: str + post_state: str + action_correct: bool + terminal_success: bool | None + + +@dataclass(frozen=True) +class TurnScore: + pre_state: str + action: str + action_correct: bool + graph_return: float + graph_advantage: float + + +def _build_rollout( + prompt_id: str, + rollout_id: str, + choices: list[str], +) -> list[Transition]: + transitions: list[Transition] = [] + state = f'{prompt_id}:s0' + for turn_id, choice in enumerate(choices): + correct = choice == 'ok' + if correct: + if turn_id == 2: + post_state = f'{prompt_id}:success' + else: + post_state = f'{prompt_id}:s{turn_id + 1}' + else: + post_state = f'{prompt_id}:dead{turn_id}' + transitions.append( + Transition( + prompt_id=prompt_id, + rollout_id=rollout_id, + turn_id=turn_id, + pre_state=state, + action=choice, + post_state=post_state, + action_correct=correct, + terminal_success=True if correct and turn_id == 2 else None, + )) + if not correct: + break + state = post_state + return transitions + + +def _build_fixture() -> list[Transition]: + paths = ( + ('ok', 'ok', 'ok'), + ('bad',), + ('ok', 'bad'), + ('ok', 'ok', 'bad'), + ('ok', 'ok', 'ok'), + ('bad',), + ('ok', 'bad'), + ('ok', 'ok', 'bad'), + ) + records: list[Transition] = [] + for prompt_index in range(8): + prompt_id = f'p{prompt_index}' + for rollout_index, choices in enumerate(paths): + records.extend(_build_rollout(prompt_id, f'r{rollout_index}', list(choices))) + return records + + +def _graph_distances(records: Iterable[Transition]) -> dict[str, int]: + edges: dict[str, set[str]] = defaultdict(set) + terminal_success: set[str] = set() + nodes: set[str] = set() + for record in records: + nodes.update((record.pre_state, record.post_state)) + edges[record.pre_state].add(record.post_state) + if record.terminal_success is True: + terminal_success.add(record.post_state) + + distances = {node: math.inf for node in nodes} + queue: deque[str] = deque() + reverse: dict[str, set[str]] = defaultdict(set) + for source, targets in edges.items(): + for target in targets: + reverse[target].add(source) + for node in terminal_success: + distances[node] = 0 + queue.append(node) + while queue: + child = queue.popleft() + for parent in reverse.get(child, ()): + candidate = distances[child] + 1 + if candidate < distances[parent]: + distances[parent] = candidate + queue.append(parent) + return distances + + +def _score_graph(records: list[Transition], gamma: float = 0.1) -> list[TurnScore]: + distances = _graph_distances(records) + raw: list[tuple[Transition, float]] = [] + for record in records: + distance = distances.get(record.post_state, math.inf) + graph_return = 0.0 if math.isinf(distance) else 10.0 * gamma**distance + raw.append((record, graph_return)) + + groups: dict[tuple[str, str], list[tuple[Transition, float]]] = defaultdict(list) + for record, graph_return in raw: + groups[(record.prompt_id, record.pre_state)].append((record, graph_return)) + + scores: list[TurnScore] = [] + for (_, _), group in groups.items(): + values = [value for _, value in group] + mean = statistics.fmean(values) + std = statistics.stdev(values) if len(values) > 1 else 0.0 + for record, graph_return in group: + advantage = 0.0 if std == 0.0 else (graph_return - mean) / std + scores.append( + TurnScore( + pre_state=record.pre_state, + action=record.action, + action_correct=record.action_correct, + graph_return=graph_return, + graph_advantage=advantage, + )) + return scores + + +def _ranking_accuracy(scores: list[TurnScore]) -> float: + groups: dict[str, list[TurnScore]] = defaultdict(list) + for score in scores: + groups[score.pre_state].append(score) + comparisons = 0 + correct = 0 + for group in groups.values(): + good = [score.graph_advantage for score in group if score.action_correct] + bad = [score.graph_advantage for score in group if not score.action_correct] + if not good or not bad: + continue + comparisons += 1 + correct += statistics.fmean(good) > statistics.fmean(bad) + return correct / comparisons if comparisons else 0.0 + + +def _sample_std(values: list[float]) -> float: + return statistics.stdev(values) if len(values) > 1 else 0.0 + + +def _episode_advantages(records: list[Transition]) -> dict[str, float]: + rollouts: dict[tuple[str, str], list[Transition]] = defaultdict(list) + for record in records: + rollouts[(record.prompt_id, record.rollout_id)].append(record) + by_prompt: dict[str, list[tuple[str, float]]] = defaultdict(list) + for (prompt_id, rollout_id), turns in rollouts.items(): + reward = 1.0 if turns[-1].terminal_success is True else 0.0 + by_prompt[prompt_id].append((rollout_id, reward)) + advantages: dict[str, float] = {} + for values in by_prompt.values(): + mean = statistics.fmean(reward for _, reward in values) + std = _sample_std([reward for _, reward in values]) + for rollout_id, reward in values: + advantages[rollout_id] = 0.0 if std == 0.0 else (reward - mean) / std + return advantages + + +def _implementation_probe(fixture: list[Transition]) -> dict: + from twinkle.advantage import StateGraphAdvantage + + by_rollout: dict[tuple[str, str], list[Transition]] = defaultdict(list) + for record in fixture: + by_rollout[(record.prompt_id, record.rollout_id)].append(record) + trajectories = [] + rewards = [] + row_by_rollout = {} + prompt_ids = sorted({record.prompt_id for record in fixture}) + for prompt_id in prompt_ids: + rollout_ids = sorted(rollout_id for candidate, rollout_id in by_rollout if candidate == prompt_id) + for rollout_id in rollout_ids: + turns = sorted(by_rollout[(prompt_id, rollout_id)], key=lambda record: record.turn_id) + row_by_rollout[(prompt_id, rollout_id)] = len(trajectories) + rewards.append(1.0 if turns[-1].terminal_success is True else 0.0) + trajectories.append({ + 'labels': [0] * 16, + 'turn_credit_status': { + 'available': True, + 'fallback_reason': None, + }, + 'turn_credits': [{ + 'turn_id': turn.turn_id, + 'action_token_start': 1 + 2 * turn.turn_id, + 'action_token_end': 2 + 2 * turn.turn_id, + 'pre_state_key': turn.pre_state, + 'post_state_key': turn.post_state, + 'policy_version': 1, + 'terminal_success': turn.terminal_success, + } for turn in turns], + }) + + # Warm up lazy torch imports outside the measured graph path. + StateGraphAdvantage(beta_episode=0.0, beta_graph=0.0)( + rewards[:8], + trajectories[:8], + num_generations=8, + ) + advantage = StateGraphAdvantage(beta_episode=0.0, beta_graph=1.0) + start = time.perf_counter() + values = advantage(rewards, trajectories, num_generations=8) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + + state_groups: dict[tuple[str, str], list[tuple[float, bool]]] = defaultdict(list) + for record in fixture: + row = row_by_rollout[(record.prompt_id, record.rollout_id)] + position = 1 + 2 * record.turn_id + state_groups[(record.prompt_id, record.pre_state)].append( + (float(values[row, position].item()), record.action_correct)) + comparisons = 0 + correct = 0 + for group in state_groups.values(): + good = [value for value, is_correct in group if is_correct] + bad = [value for value, is_correct in group if not is_correct] + if not good or not bad: + continue + comparisons += 1 + correct += statistics.fmean(good) > statistics.fmean(bad) + + return { + 'credit_ranking_accuracy': correct / comparisons if comparisons else 0.0, + 'cpu_time_ms': elapsed_ms, + 'transition_count': advantage.last_diagnostics.transition_count, + 'comparable_state_group_count': advantage.last_diagnostics.comparable_state_group_count, + 'fallback_counts': advantage.last_diagnostics.fallback_counts, + } + + +def run_gate0() -> dict: + fixture = _build_fixture() + start = time.perf_counter() + scores = _score_graph(fixture) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + implementation = _implementation_probe(fixture) + + state_coverage = sum(bool(record.pre_state and record.post_state) for record in fixture) / len(fixture) + group_sizes: dict[tuple[str, str], int] = defaultdict(int) + for record in fixture: + group_sizes[(record.prompt_id, record.pre_state)] += 1 + same_state_support = sum(size >= 2 for size in group_sizes.values()) / len(group_sizes) + rank_accuracy = _ranking_accuracy(scores) + + fallback_advantage = _episode_advantages(fixture) + valid_rollout_ids = {record.rollout_id for record in fixture} + fallback_complete = valid_rollout_ids == set(fallback_advantage) + + no_success = [ + Transition(**{**asdict(record), 'terminal_success': None}) + for record in fixture + if not record.post_state.endswith(':success') + ] + no_success_scores = _score_graph(no_success) + no_success_graph_mass = sum(abs(score.graph_advantage) for score in no_success_scores) + + thresholds = { + 'state_key_coverage': 0.95, + 'same_state_support': 0.40, + 'credit_ranking_accuracy': 0.80, + 'fallback_rate_max': 0.05, + 'cpu_time_ms_max': 100.0, + } + observed = { + 'state_key_coverage': state_coverage, + 'same_state_support': same_state_support, + 'credit_ranking_accuracy': rank_accuracy, + 'fallback_rate': 0.0, + 'cpu_time_ms': elapsed_ms, + 'implementation_credit_ranking_accuracy': implementation['credit_ranking_accuracy'], + 'implementation_cpu_time_ms': implementation['cpu_time_ms'], + } + checks = { + 'state_key_coverage': state_coverage >= thresholds['state_key_coverage'], + 'same_state_support': same_state_support >= thresholds['same_state_support'], + 'credit_ranking_accuracy': rank_accuracy >= thresholds['credit_ranking_accuracy'], + 'fallback_rate': observed['fallback_rate'] <= thresholds['fallback_rate_max'], + 'cpu_time_ms': elapsed_ms <= thresholds['cpu_time_ms_max'], + 'implementation_credit_ranking_accuracy': + implementation['credit_ranking_accuracy'] >= thresholds['credit_ranking_accuracy'], + 'implementation_cpu_time_ms': implementation['cpu_time_ms'] <= thresholds['cpu_time_ms_max'], + 'implementation_transition_count': implementation['transition_count'] == len(fixture), + 'implementation_no_fallback': implementation['fallback_counts'] == {}, + 'episode_fallback_complete': fallback_complete, + 'no_success_has_zero_graph_mass': no_success_graph_mass == 0.0, + } + return { + 'passed': all(checks.values()), + 'thresholds': thresholds, + 'observed': observed, + 'checks': checks, + 'fixture': { + 'transition_count': len(fixture), + 'state_group_count': len(group_sizes), + 'mean_group_support': statistics.fmean(group_sizes.values()), + }, + 'negative_controls': { + 'no_success_graph_mass': no_success_graph_mass, + }, + 'implementation': implementation, + 'extra_policy_forwards': 0, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--output', type=Path) + args = parser.parse_args() + result = run_gate0() + text = json.dumps(result, indent=2, ensure_ascii=False) + print(text) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text + '\n', encoding='utf-8') + return 0 if result['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/cookbook/rl/graphcredit/gate1.py b/cookbook/rl/graphcredit/gate1.py new file mode 100644 index 000000000..694daa3bc --- /dev/null +++ b/cookbook/rl/graphcredit/gate1.py @@ -0,0 +1,777 @@ +"""Matched-budget Gate 1 training for GRPO versus StateGraphAdvantage. + +The task is a deterministic two-step grid world. A wrong move ends the episode +with reward 0, while reaching the goal returns 1. GRPO therefore only sees the +final episode result. StateGraphAdvantage can additionally distinguish a useful +intermediate move from a wrong move even when the episode later fails. + +This is a mechanism study, not a production training entry point. It uses a +standalone Transformers + PEFT loop so both arms share the same rollout, +policy-gradient update and token budget. +""" + +from __future__ import annotations + +import argparse +import json +import random +import re +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Sequence + +import numpy as np +import torch + +from twinkle.advantage import GRPOAdvantage, StateGraphAdvantage + + +ACTIONS = ('up', 'down', 'left', 'right') +MOVES = { + 'up': (-1, 0), + 'down': (1, 0), + 'left': (0, -1), + 'right': (0, 1), +} +ACTION_PATTERN = re.compile(r'\s*(up|down|left|right)\s*', re.IGNORECASE) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument('--model-path', required=True) + parser.add_argument('--mode', choices=('grpo', 'graphcredit'), required=True) + parser.add_argument('--output-dir', required=True) + parser.add_argument('--device', default='npu:0') + parser.add_argument('--seed', type=int, default=20261118) + parser.add_argument('--steps', type=int, default=20) + parser.add_argument('--eval-tasks', type=int, default=24) + parser.add_argument('--eval-every', type=int, default=0) + parser.add_argument('--group-size', type=int, default=8) + parser.add_argument('--distance', type=int, default=2) + parser.add_argument('--grid-size', type=int, default=5) + parser.add_argument('--max-new-tokens', type=int, default=24) + parser.add_argument('--max-length', type=int, default=1024) + parser.add_argument('--learning-rate', type=float, default=5e-5) + parser.add_argument('--lora-rank', type=int, default=8) + parser.add_argument('--lora-alpha', type=int, default=16) + parser.add_argument('--temperature', type=float, default=1.2) + parser.add_argument('--top-p', type=float, default=0.95) + parser.add_argument('--group-retries', type=int, default=12) + parser.add_argument('--max-step-attempts', type=int, default=6) + parser.add_argument('--beta-episode', type=float, default=1.0) + parser.add_argument('--beta-graph', type=float, default=1.0) + parser.add_argument('--gamma', type=float, default=0.5) + parser.add_argument('--min-state-support', type=int, default=2) + parser.add_argument('--max-grad-norm', type=float, default=1.0) + return parser.parse_args() + + +def seed_everything(seed: int) -> None: + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if hasattr(torch, 'npu'): + torch.npu.manual_seed_all(seed) + + +def ensure_npu(device_name: str) -> None: + if device_name.startswith('npu'): + import torch_npu # noqa: F401 + + +def reset_peak_hbm() -> None: + if hasattr(torch, 'npu') and torch.npu.is_available(): + torch.npu.reset_peak_memory_stats() + elif torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats() + + +def peak_hbm_gb() -> float: + if hasattr(torch, 'npu') and torch.npu.is_available(): + return float(torch.npu.max_memory_allocated() / 2**30) + if torch.cuda.is_available(): + return float(torch.cuda.max_memory_allocated() / 2**30) + return 0.0 + + +def synchronize(device: torch.device) -> None: + if device.type == 'npu': + torch.npu.synchronize() + elif device.type == 'cuda': + torch.cuda.synchronize() + + +@dataclass(frozen=True) +class GridTask: + task_uid: int + grid_size: int + distance: int + start: tuple[int, int] + goal: tuple[int, int] + + @property + def task_key(self) -> str: + return f'task-{self.task_uid}' + + def state_key(self, position: tuple[int, int]) -> str: + return f'{self.task_key}:{position[0]}:{position[1]}' + + @property + def dead_state_key(self) -> str: + return f'{self.task_key}:dead' + + +def make_task(task_uid: int, *, grid_size: int, distance: int) -> GridTask: + if distance < 1: + raise ValueError('distance must be positive') + if grid_size <= distance: + raise ValueError('grid_size must be larger than distance') + rng = random.Random(task_uid) + for _ in range(10_000): + start = (rng.randrange(grid_size), rng.randrange(grid_size)) + axis = rng.choice(('row', 'col')) + sign = rng.choice((-1, 1)) + goal = ( + start[0] + sign * distance if axis == 'row' else start[0], + start[1] + sign * distance if axis == 'col' else start[1], + ) + if 0 <= goal[0] < grid_size and 0 <= goal[1] < grid_size: + return GridTask(task_uid, grid_size, distance, start, goal) + raise RuntimeError(f'could not generate task {task_uid}') + + +def format_action(action: str) -> str: + return f'{action}' + + +def parse_action(text: str) -> str | None: + matches = ACTION_PATTERN.findall(text) + if len(matches) == 1: + return matches[0].lower() + stripped = text.strip().lower() + return stripped if stripped in ACTIONS else None + + +def render_prompt(task: GridTask, position: tuple[int, int], history: Sequence[str]) -> str: + row, col = position + goal_row, goal_col = task.goal + distance = abs(goal_row - row) + abs(goal_col - col) + history_text = ' -> '.join(history) if history else '' + return ( + 'You control a robot on a rectangular grid. Do not explain or reason aloud. ' + 'Reply with exactly one action tag: up, down, ' + 'left, or right.\n' + 'Coordinates are zero-based (row, column). Up decreases row, down increases row, ' + 'left decreases column and right increases column. A move that does not reduce ' + 'the Manhattan distance to the goal immediately ends the episode.\n' + f'Grid size: {task.grid_size}x{task.grid_size}.\n' + f'Current position: ({row}, {col}).\n' + f'Goal position: ({goal_row}, {goal_col}).\n' + f'Remaining Manhattan distance: {distance}.\n' + f'Action history: {history_text}.\n' + 'Choose the next action.' + ) + + +def render_messages(prompt: str) -> list[dict[str, str]]: + return [ + { + 'role': 'system', + 'content': 'You are a precise grid-navigation policy.' + }, + { + 'role': 'user', + 'content': prompt + }, + ] + + +@dataclass +class EpisodeTurn: + prompt: str + action: str + pre_state_key: str + post_state_key: str + action_token_start: int + action_token_end: int + terminal_success: bool | None + + +@dataclass +class Episode: + task: GridTask + uid: str + position: tuple[int, int] + done: bool = False + success: bool = False + reward: float = 0.0 + invalid_action: bool = False + input_ids: list[int] = field(default_factory=list) + turns: list[EpisodeTurn] = field(default_factory=list) + + @property + def history(self) -> list[str]: + return [turn.action for turn in self.turns] + + def to_trajectory(self, policy_version: int) -> dict[str, Any]: + records = [ + { + 'turn_id': index, + 'action_token_start': turn.action_token_start, + 'action_token_end': turn.action_token_end, + 'observation_token_start': None, + 'observation_token_end': None, + 'pre_state_key': turn.pre_state_key, + 'post_state_key': turn.post_state_key, + 'tool_events': [], + 'policy_version': policy_version, + 'terminal_success': turn.terminal_success, + } + for index, turn in enumerate(self.turns) + ] + return { + 'input_ids': list(self.input_ids), + 'turn_credits': records, + 'turn_credit_status': { + 'available': bool(records) and not self.invalid_action, + 'fallback_reason': None if records else 'turn_credit_empty', + }, + 'episode_reward': self.reward, + 'terminal_success': self.success, + } + + +def step_episode(episode: Episode, action: str) -> None: + if episode.done: + raise RuntimeError('cannot step a finished episode') + if action not in MOVES: + raise ValueError(f'unsupported action: {action}') + + row, col = episode.position + delta_row, delta_col = MOVES[action] + next_position = (row + delta_row, col + delta_col) + current_distance = abs(row - episode.task.goal[0]) + abs(col - episode.task.goal[1]) + next_distance = ( + abs(next_position[0] - episode.task.goal[0]) + + abs(next_position[1] - episode.task.goal[1]) + ) + out_of_bounds = not ( + 0 <= next_position[0] < episode.task.grid_size + and 0 <= next_position[1] < episode.task.grid_size + ) + if out_of_bounds or next_distance >= current_distance: + episode.done = True + episode.success = False + episode.reward = 0.0 + episode.position = next_position + return + + episode.position = next_position + if next_position == episode.task.goal: + episode.done = True + episode.success = True + episode.reward = 1.0 + + +def _append_prompt_action( + episode: Episode, + prompt_ids: Sequence[int], + action_ids: Sequence[int], + *, + eos_token_id: int | None, +) -> tuple[int, int]: + before = len(episode.input_ids) + episode.input_ids.extend(int(token) for token in prompt_ids) + action_start = before + len(prompt_ids) - 1 + episode.input_ids.extend(int(token) for token in action_ids) + action_end = action_start + len(action_ids) + if eos_token_id is not None: + episode.input_ids.append(int(eos_token_id)) + return action_start, action_end + + +def rollout_group( + policy: Any, + tokenizer: Any, + task: GridTask, + *, + group_size: int, + max_new_tokens: int, + max_length: int, + temperature: float, + top_p: float, + device: torch.device, + sample: bool, +) -> list[Episode]: + episodes = [ + Episode(task=task, uid=f'{task.task_uid}:{index}', position=task.start) + for index in range(group_size) + ] + for _turn in range(1, task.distance + 1): + active = [episode for episode in episodes if not episode.done] + if not active: + break + prompts = [render_prompt(task, episode.position, episode.history) for episode in active] + outputs = generate_actions( + policy, + tokenizer, + prompts, + device, + sample=sample, + max_new_tokens=max_new_tokens, + temperature=temperature, + top_p=top_p, + ) + for episode, prompt, raw_output in zip(active, prompts, outputs): + action = parse_action(raw_output) + if action is None: + episode.invalid_action = True + episode.done = True + episode.reward = 0.0 + continue + prompt_text = render_chat_prompt(tokenizer, prompt) + prompt_ids = encode_text(tokenizer, prompt_text) + action_ids = encode_text(tokenizer, format_action(action)) + if len(episode.input_ids) + len(prompt_ids) + len(action_ids) > max_length: + episode.invalid_action = True + episode.done = True + episode.reward = 0.0 + continue + pre_state_key = task.state_key(episode.position) + action_start, action_end = _append_prompt_action( + episode, + prompt_ids, + action_ids, + eos_token_id=tokenizer.eos_token_id, + ) + step_episode(episode, action) + post_state_key = ( + task.state_key(episode.position) if episode.success or not episode.done else task.dead_state_key + ) + episode.turns.append( + EpisodeTurn( + prompt=prompt, + action=action, + pre_state_key=pre_state_key, + post_state_key=post_state_key, + action_token_start=action_start, + action_token_end=action_end, + terminal_success=True if episode.success else (False if episode.done else None), + )) + for episode in episodes: + if not episode.done: + episode.done = True + episode.reward = 0.0 + return episodes + + +def render_chat_prompt(tokenizer: Any, prompt: str) -> str: + return tokenizer.apply_chat_template( + render_messages(prompt), + tokenize=False, + add_generation_prompt=True, + enable_thinking=False, + ) + + +def encode_text(tokenizer: Any, text: str) -> list[int]: + return list(tokenizer(text, add_special_tokens=False, truncation=False)['input_ids']) + + +def generate_actions( + policy: Any, + tokenizer: Any, + prompts: Sequence[str], + device: torch.device, + *, + sample: bool, + max_new_tokens: int, + temperature: float, + top_p: float, +) -> list[str]: + rendered = [render_chat_prompt(tokenizer, prompt) for prompt in prompts] + encoded = tokenizer( + rendered, + padding=True, + return_tensors='pt', + add_special_tokens=False, + ) + encoded = {key: value.to(device) for key, value in encoded.items()} + policy.eval() + policy.config.use_cache = True + kwargs: dict[str, Any] = { + 'max_new_tokens': max_new_tokens, + 'do_sample': sample, + 'pad_token_id': tokenizer.pad_token_id, + 'eos_token_id': tokenizer.eos_token_id, + 'use_cache': True, + } + if sample: + kwargs.update(temperature=temperature, top_p=top_p) + with torch.inference_mode(): + generated = policy.generate(**encoded, **kwargs) + width = encoded['input_ids'].shape[1] + return tokenizer.batch_decode(generated[:, width:], skip_special_tokens=True) + + +def compute_advantages( + mode: str, + episodes: Sequence[Episode], + *, + policy_version: int, + group_size: int, + args: argparse.Namespace, +) -> tuple[torch.Tensor, dict[str, Any]]: + rewards = [episode.reward for episode in episodes] + trajectories = [episode.to_trajectory(policy_version) for episode in episodes] + max_length = max((len(episode.input_ids) for episode in episodes), default=0) + if mode == 'grpo': + episode_advantage = GRPOAdvantage()(rewards, num_generations=group_size, scale='group') + advantages = episode_advantage.unsqueeze(1).repeat(1, max_length) + diagnostics = {'mode': 'grpo'} + return advantages, diagnostics + + graph = StateGraphAdvantage( + beta_episode=args.beta_episode, + beta_graph=args.beta_graph, + gamma=args.gamma, + min_state_support=args.min_state_support, + ) + advantages = graph(rewards, trajectories, num_generations=group_size) + diagnostics = { + 'mode': 'graphcredit', + 'transition_count': graph.last_diagnostics.transition_count, + 'graph_node_count': graph.last_diagnostics.graph_node_count, + 'graph_edge_count': graph.last_diagnostics.graph_edge_count, + 'comparable_state_group_count': graph.last_diagnostics.comparable_state_group_count, + 'fallback_counts': graph.last_diagnostics.fallback_counts, + } + return advantages, diagnostics + + +def pad_episode_batch( + episodes: Sequence[Episode], + pad_token_id: int, +) -> tuple[torch.Tensor, torch.Tensor]: + max_length = max(len(episode.input_ids) for episode in episodes) + rows = [] + masks = [] + for episode in episodes: + padding = max_length - len(episode.input_ids) + rows.append(episode.input_ids + [pad_token_id] * padding) + masks.append([1] * len(episode.input_ids) + [0] * padding) + return ( + torch.tensor(rows, dtype=torch.long), + torch.tensor(masks, dtype=torch.long), + ) + + +def policy_gradient_update( + policy: Any, + optimizer: torch.optim.Optimizer, + episodes: Sequence[Episode], + advantages: torch.Tensor, + *, + device: torch.device, + pad_token_id: int, + max_grad_norm: float, +) -> tuple[float, int]: + policy.train() + policy.config.use_cache = False + optimizer.zero_grad(set_to_none=True) + input_ids, attention_mask = pad_episode_batch(episodes, pad_token_id) + input_ids = input_ids.to(device) + attention_mask = attention_mask.to(device) + outputs = policy( + input_ids=input_ids, + attention_mask=attention_mask, + use_cache=False, + ) + logits = outputs.logits[:, :-1, :].float() + targets = input_ids[:, 1:] + token_logps = torch.log_softmax(logits, dim=-1).gather( + -1, + targets.unsqueeze(-1), + ).squeeze(-1) + advantages = advantages.to(device=device, dtype=token_logps.dtype) + + loss = input_ids.new_zeros((), dtype=torch.float32) + action_tokens = 0 + for row, episode in enumerate(episodes): + for turn in episode.turns: + start = turn.action_token_start + end = turn.action_token_end + if start < 0 or end <= start or end > token_logps.shape[1]: + raise RuntimeError( + f'invalid action span for {episode.uid}: [{start}, {end}) ' + f'with sequence width {token_logps.shape[1]}') + loss = loss - ( + token_logps[row, start:end] * advantages[row, start:end] + ).sum() + action_tokens += end - start + if action_tokens == 0: + raise RuntimeError('episode group contains no trainable action tokens') + loss = loss / action_tokens + loss.backward() + torch.nn.utils.clip_grad_norm_( + [parameter for parameter in policy.parameters() if parameter.requires_grad], + max_grad_norm, + ) + optimizer.step() + return float(loss.detach().cpu()), action_tokens + + +def evaluate( + policy: Any, + tokenizer: Any, + args: argparse.Namespace, + device: torch.device, + *, + split_offset: int, +) -> dict[str, float]: + successes = 0 + invalid = 0 + turns = 0 + rewards = [] + started = time.perf_counter() + for index in range(args.eval_tasks): + task = make_task( + split_offset + index, + grid_size=args.grid_size, + distance=args.distance, + ) + episode = rollout_group( + policy, + tokenizer, + task, + group_size=1, + max_new_tokens=args.max_new_tokens, + max_length=args.max_length, + temperature=args.temperature, + top_p=args.top_p, + device=device, + sample=False, + )[0] + successes += int(episode.success) + invalid += int(episode.invalid_action) + turns += len(episode.turns) + rewards.append(episode.reward) + total = max(1, len(rewards)) + return { + 'success_rate': successes / total, + 'invalid_rate': invalid / total, + 'reward_mean': float(np.mean(rewards)) if rewards else 0.0, + 'mean_turns': turns / total, + 'eval_seconds': time.perf_counter() - started, + } + + +def sample_training_group( + policy: Any, + tokenizer: Any, + args: argparse.Namespace, + device: torch.device, + *, + step: int, +) -> tuple[GridTask, list[Episode]] | None: + for attempt in range(args.group_retries): + task_uid = args.seed + 10_000 + step * 100 + attempt + task = make_task(task_uid, grid_size=args.grid_size, distance=args.distance) + episodes = rollout_group( + policy, + tokenizer, + task, + group_size=args.group_size, + max_new_tokens=args.max_new_tokens, + max_length=args.max_length, + temperature=args.temperature, + top_p=args.top_p, + device=device, + sample=True, + ) + if any(episode.invalid_action for episode in episodes): + continue + rewards = [episode.reward for episode in episodes] + if not any(rewards) or all(rewards): + continue + return task, episodes + return None + + +def load_tokenizer(model_path: str) -> Any: + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + tokenizer.padding_side = 'left' + return tokenizer + + +def load_policy(model_path: str, args: argparse.Namespace, device: torch.device) -> Any: + from peft import LoraConfig, get_peft_model + from transformers import AutoModelForCausalLM + + policy = AutoModelForCausalLM.from_pretrained( + model_path, + dtype=torch.bfloat16, + trust_remote_code=True, + low_cpu_mem_usage=True, + ).to(device) + if hasattr(policy, 'gradient_checkpointing_enable'): + policy.gradient_checkpointing_enable() + policy = get_peft_model( + policy, + LoraConfig( + task_type='CAUSAL_LM', + r=args.lora_rank, + lora_alpha=args.lora_alpha, + lora_dropout=0.05, + target_modules=[ + 'q_proj', + 'k_proj', + 'v_proj', + 'o_proj', + 'gate_proj', + 'up_proj', + 'down_proj', + ], + ), + ) + policy.config.use_cache = False + return policy + + +def train(policy: Any, tokenizer: Any, args: argparse.Namespace, device: torch.device) -> dict[str, Any]: + trainable = [parameter for parameter in policy.parameters() if parameter.requires_grad] + optimizer = torch.optim.AdamW(trainable, lr=args.learning_rate) + initial_eval = evaluate( + policy, + tokenizer, + args, + device, + split_offset=args.seed + 500_000, + ) + reset_peak_hbm() + history = [] + completed_steps = 0 + attempted_steps = 0 + max_attempts = args.steps * args.max_step_attempts + while completed_steps < args.steps and attempted_steps < max_attempts: + attempted_steps += 1 + step_started = time.perf_counter() + sampled = sample_training_group( + policy, + tokenizer, + args, + device, + step=attempted_steps - 1, + ) + if sampled is None: + history.append({ + 'attempt': attempted_steps, + 'completed_steps': completed_steps, + 'skipped': True, + }) + continue + task, episodes = sampled + synchronize(device) + rollout_seconds = time.perf_counter() - step_started + rewards = [episode.reward for episode in episodes] + advantages, diagnostics = compute_advantages( + args.mode, + episodes, + policy_version=completed_steps + 1, + group_size=args.group_size, + args=args, + ) + loss, action_tokens = policy_gradient_update( + policy, + optimizer, + episodes, + advantages, + device=device, + pad_token_id=tokenizer.pad_token_id, + max_grad_norm=args.max_grad_norm, + ) + synchronize(device) + record = { + 'step': completed_steps + 1, + 'attempt': attempted_steps, + 'task_uid': task.task_uid, + 'reward_mean': float(np.mean(rewards)), + 'success_rate': float(np.mean([episode.success for episode in episodes])), + 'mean_actions': float(np.mean([len(episode.turns) for episode in episodes])), + 'loss': loss, + 'action_tokens': action_tokens, + 'rollout_seconds': rollout_seconds, + 'step_seconds': time.perf_counter() - step_started, + 'peak_hbm_gb': peak_hbm_gb(), + **diagnostics, + } + history.append(record) + print(json.dumps(record, ensure_ascii=False), flush=True) + completed_steps += 1 + if args.eval_every > 0 and completed_steps % args.eval_every == 0: + heldout = evaluate( + policy, + tokenizer, + args, + device, + split_offset=args.seed + 500_000, + ) + record['heldout_eval'] = heldout + print( + json.dumps( + { + 'step': completed_steps, + 'heldout_success_rate': heldout['success_rate'], + 'heldout_invalid_rate': heldout['invalid_rate'], + }, + ensure_ascii=False, + ), + flush=True, + ) + final_eval = evaluate( + policy, + tokenizer, + args, + device, + split_offset=args.seed + 500_000, + ) + return { + 'mode': args.mode, + 'history': history, + 'completed_steps': completed_steps, + 'attempted_steps': attempted_steps, + 'skipped_steps': attempted_steps - completed_steps, + 'initial_eval': initial_eval, + 'final_eval': final_eval, + 'peak_hbm_gb': peak_hbm_gb(), + } + + +def main() -> None: + args = parse_args() + ensure_npu(args.device) + seed_everything(args.seed) + device = torch.device(args.device) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + tokenizer = load_tokenizer(args.model_path) + policy = load_policy(args.model_path, args, device) + started = time.perf_counter() + result = train(policy, tokenizer, args, device) + result['wall_seconds'] = time.perf_counter() - started + result['args'] = vars(args) + policy.save_pretrained(output_dir / 'adapter') + (output_dir / 'result.json').write_text( + json.dumps(result, indent=2, ensure_ascii=False), + encoding='utf-8', + ) + print(json.dumps(result, indent=2, ensure_ascii=False), flush=True) + + +if __name__ == '__main__': + main() diff --git a/cookbook/rl/graphcredit/run_gate1_a2.sh b/cookbook/rl/graphcredit/run_gate1_a2.sh new file mode 100644 index 000000000..002fff9fb --- /dev/null +++ b/cookbook/rl/graphcredit/run_gate1_a2.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +set -eo pipefail + +MODE="${1:?usage: run_gate1_a2.sh [steps] [eval-tasks] [output-dir]}" +STEPS="${2:-20}" +EVAL_TASKS="${3:-24}" +OUTPUT_DIR="${4:-outputs/graphcredit-gate1/${MODE}}" + +MODEL_PATH="${MODEL_PATH:-/workspace/shared_assets/models/Qwen/Qwen3-0.6B}" +SEED="${SEED:-20261118}" +GROUP_SIZE="${GROUP_SIZE:-8}" +MAX_NEW_TOKENS="${MAX_NEW_TOKENS:-24}" +MAX_LENGTH="${MAX_LENGTH:-1024}" +LEARNING_RATE="${LEARNING_RATE:-5e-5}" +DISTANCE="${DISTANCE:-2}" +GRID_SIZE="${GRID_SIZE:-5}" +EVAL_EVERY="${EVAL_EVERY:-0}" +MAX_STEP_ATTEMPTS="${MAX_STEP_ATTEMPTS:-6}" + +source /usr/local/Ascend/ascend-toolkit/set_env.sh +set -u +export PYTHONPATH="/workspace/graphcredit-gate0/src:/workspace/graphcredit-deps${PYTHONPATH:+:${PYTHONPATH}}" + +cd /workspace/graphcredit-gate0 +exec /workspace/npu-ai-env/bin/python cookbook/rl/graphcredit/gate1.py \ + --model-path "${MODEL_PATH}" \ + --mode "${MODE}" \ + --steps "${STEPS}" \ + --eval-tasks "${EVAL_TASKS}" \ + --eval-every "${EVAL_EVERY}" \ + --group-size "${GROUP_SIZE}" \ + --max-step-attempts "${MAX_STEP_ATTEMPTS}" \ + --distance "${DISTANCE}" \ + --grid-size "${GRID_SIZE}" \ + --max-new-tokens "${MAX_NEW_TOKENS}" \ + --max-length "${MAX_LENGTH}" \ + --learning-rate "${LEARNING_RATE}" \ + --seed "${SEED}" \ + --device npu:0 \ + --output-dir "${OUTPUT_DIR}" diff --git "a/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/StateGraphAdvantage.md" "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/StateGraphAdvantage.md" new file mode 100644 index 000000000..681645656 --- /dev/null +++ "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/StateGraphAdvantage.md" @@ -0,0 +1,72 @@ +# StateGraphAdvantage + +`StateGraphAdvantage` 在 episode-level GRPO advantage 之上,增加基于显式状态图的 +turn-level advantage。它依赖阶段 A 产生的 `turn_credits`,不根据聊天文本推断状态, +也不会增加额外 policy forward。 + +## 使用示例 + +```python +from twinkle.advantage import StateGraphAdvantage + +advantage_fn = StateGraphAdvantage( + beta_episode=1.0, + beta_graph=1.0, + gamma=0.1, + min_state_support=2, +) + +advantages = advantage_fn( + rewards, + trajectories, + num_generations=4, +) +``` + +每条 trajectory 需要包含: + +- `turn_credit_status.available=True`; +- `turn_credits` 中每个 turn 的 `pre_state_key`、`post_state_key`; +- 左闭右开的 `action_token_start` / `action_token_end`; +- 明确的 `policy_version`; +- 至少一个带 `terminal_success=True` 的状态边。 + +## 计算方式 + +同一 prompt group 内,状态键构成图节点,turn 构成有向边。成功终态的距离为 0, +算法从成功态反向计算最短距离,并按相同 `pre_state_key` 对动作做归一化: + +```text +R_t^G = 10 * gamma ^ distance(post_state_t) +A_t^G = Normalize(R_t^G | same prompt group, same pre_state_key) +A_t = beta_episode * A_t^E + beta_graph * A_t^G +``` + +`A_t^E` 每条 episode 只计算一次,然后广播到该 episode 的可训练 token。 +graph advantage 只写入 turn 的 action span,不覆盖 observation、bridge 或 padding。 + +## 严格回退 + +以下情况不会部分解释 graph advantage,而是保留原 GRPO 数值: + +- 缺失状态键或 terminal success; +- 同一轮工具结果的状态或成功标记冲突; +- action span、observation span 越界; +- policy version 缺失、冲突或类型非法; +- 没有成功终态; +- 状态图节点或边超过预算; +- turn credit 不可用或为空。 + +回退原因记录在 `advantage_fn.last_diagnostics.fallback_counts`。相同状态的动作 +少于 `min_state_support` 时,该状态组保持零 graph advantage。 + +## Gate 0 + +确定性探针位于 `cookbook/rl/graphcredit/gate0.py`: + +```bash +PYTHONPATH=src python cookbook/rl/graphcredit/gate0.py +``` + +探针检查状态覆盖率、同状态支撑、动作排序、无成功终态负例、回退完整性、 +CPU 计算耗时以及额外 policy forward 数量。 diff --git "a/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst" "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst" index f1522152d..138837809 100644 --- "a/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst" +++ "b/docs/source_zh/\347\273\204\344\273\266/\344\274\230\345\212\277/index.rst" @@ -7,3 +7,4 @@ GRPOAdvantage.md GroupAdmissionPolicy.md RLOOAdvantage.md + StateGraphAdvantage.md diff --git a/src/twinkle/advantage/__init__.py b/src/twinkle/advantage/__init__.py index 519dd1fe4..72ded0894 100644 --- a/src/twinkle/advantage/__init__.py +++ b/src/twinkle/advantage/__init__.py @@ -3,10 +3,12 @@ from .gae import GAEAdvantage from .grpo import GRPOAdvantage from .rloo import RLOOAdvantage +from .state_graph import StateGraphAdvantage __all__ = [ 'Advantage', 'GAEAdvantage', 'GRPOAdvantage', 'RLOOAdvantage', + 'StateGraphAdvantage', ] diff --git a/src/twinkle/advantage/state_graph.py b/src/twinkle/advantage/state_graph.py new file mode 100644 index 000000000..d4515bd11 --- /dev/null +++ b/src/twinkle/advantage/state_graph.py @@ -0,0 +1,284 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""State-graph turn credit with episode-level GRPO as the safe fallback.""" + +from __future__ import annotations + +import math +import statistics +from collections import defaultdict, deque +from dataclasses import dataclass, field +from typing import Any, Mapping, Sequence + +from .base import Advantage +from .grpo import GRPOAdvantage + +__all__ = ['StateGraphAdvantage', 'StateGraphAdvantageDiagnostics'] + + +@dataclass +class StateGraphAdvantageDiagnostics: + transition_count: int = 0 + graph_node_count: int = 0 + graph_edge_count: int = 0 + comparable_state_group_count: int = 0 + fallback_counts: dict[str, int] = field(default_factory=dict) + output_shape: tuple[int, int] = (0, 0) + + def add_fallback(self, reason: str) -> None: + self.fallback_counts[reason] = self.fallback_counts.get(reason, 0) + 1 + + +class StateGraphAdvantage(Advantage): + """Combine GRPO episode advantages with state-conditioned graph returns. + + ``turn_credits`` are emitted by ``MultiTurnRollout`` and use labels-space, + half-open action spans. A group is graph-scored only when every participating + policy version matches, at least one explicit success state exists, and the + graph stays within the configured budget. Otherwise its rows remain exactly + equal to the episode-level GRPO advantage. + """ + + def __init__( + self, + *, + beta_episode: float = 1.0, + beta_graph: float = 1.0, + gamma: float = 0.1, + min_state_support: int = 2, + max_graph_nodes: int = 4096, + max_graph_edges: int = 16384, + epsilon: float = 1e-8, + ): + if not math.isfinite(beta_episode) or beta_episode < 0: + raise ValueError('beta_episode must be finite and non-negative') + if not math.isfinite(beta_graph) or beta_graph < 0: + raise ValueError('beta_graph must be finite and non-negative') + if not 0 < gamma <= 1: + raise ValueError('gamma must be in (0, 1]') + if min_state_support < 2: + raise ValueError('min_state_support must be at least two') + if max_graph_nodes < 1 or max_graph_edges < 1: + raise ValueError('graph budgets must be positive') + if epsilon <= 0: + raise ValueError('epsilon must be positive') + self.beta_episode = beta_episode + self.beta_graph = beta_graph + self.gamma = gamma + self.min_state_support = min_state_support + self.max_graph_nodes = max_graph_nodes + self.max_graph_edges = max_graph_edges + self.epsilon = epsilon + self.last_diagnostics: StateGraphAdvantageDiagnostics | None = None + + def compute( + self, + rewards: Sequence[float], + trajectories: Sequence[Mapping[str, Any]], + *, + num_generations: int, + sequence_lengths: Sequence[int] | None = None, + ) -> tuple[Any, StateGraphAdvantageDiagnostics]: + import torch + + if num_generations <= 1: + raise ValueError('StateGraphAdvantage requires at least two generations per prompt') + if len(trajectories) != len(rewards) or len(rewards) % num_generations: + raise ValueError('rewards and trajectories must contain complete groups') + if sequence_lengths is None: + lengths = [self._sequence_length(trajectory) for trajectory in trajectories] + else: + lengths = [int(length) for length in sequence_lengths] + if len(lengths) != len(trajectories) or any(length <= 0 for length in lengths): + raise ValueError('sequence_lengths must be positive and match trajectories') + + episode = GRPOAdvantage()(torch.as_tensor(rewards, dtype=torch.float32), num_generations=num_generations) + advantages = (self.beta_episode * episode).unsqueeze(1).repeat(1, max(lengths)) + diagnostics = StateGraphAdvantageDiagnostics(output_shape=tuple(advantages.shape)) + + for group_start in range(0, len(trajectories), num_generations): + group_rows = range(group_start, group_start + num_generations) + records = [] + fallback_rows: set[int] = set() + versions: set[int | str] = set() + versioned_records = 0 + for row in group_rows: + trajectory = trajectories[row] + status = trajectory.get('turn_credit_status') or {} + if status.get('available') is not True: + fallback_rows.add(row) + diagnostics.add_fallback(str(status.get('fallback_reason') or 'turn_credit_unavailable')) + continue + credits = list(trajectory.get('turn_credits') or []) + if not credits: + fallback_rows.add(row) + diagnostics.add_fallback('turn_credit_empty') + continue + for record in credits: + try: + parsed = self._validate_record(record, lengths[row]) + except (TypeError, ValueError) as exc: + fallback_rows.add(row) + diagnostics.add_fallback(str(exc) or 'turn_credit_invalid') + break + records.append((row, parsed)) + if parsed.get('policy_version') is not None: + versions.add(parsed['policy_version']) + versioned_records += 1 + + if len(versions) > 1: + diagnostics.add_fallback('policy_version_mismatch') + continue + if not records: + continue + if versioned_records != len(records): + diagnostics.add_fallback('policy_version_missing') + continue + + valid_records = [(row, record) for row, record in records if row not in fallback_rows] + if not valid_records: + continue + try: + distances, node_count, edge_count = self._distances(valid_records) + except ValueError as exc: + diagnostics.add_fallback(str(exc)) + continue + diagnostics.graph_node_count += node_count + diagnostics.graph_edge_count += edge_count + diagnostics.transition_count += len(valid_records) + + state_groups: dict[str, list[tuple[int, Mapping[str, Any], float]]] = defaultdict(list) + for row, record in valid_records: + distance = distances.get(record['post_state_key'], math.inf) + graph_return = 0.0 if math.isinf(distance) else 10.0 * self.gamma**distance + state_groups[record['pre_state_key']].append((row, record, graph_return)) + + for group in state_groups.values(): + if len(group) < self.min_state_support: + continue + values = [value for _, _, value in group] + mean = statistics.fmean(values) + std = statistics.stdev(values) + if std <= self.epsilon: + continue + diagnostics.comparable_state_group_count += 1 + for row, record, value in group: + local = (value - mean) / std + start, end = record['action_token_start'], record['action_token_end'] + advantages[row, start:end] += self.beta_graph * local + + return advantages, diagnostics + + @staticmethod + def _sequence_length(trajectory: Mapping[str, Any]) -> int: + for key in ('labels', 'input_ids', 'completion_mask'): + value = trajectory.get(key) + if value is not None: + return len(value) + raise ValueError('trajectory needs labels, input_ids, or completion_mask to size advantages') + + @staticmethod + def _validate_record(record: Any, sequence_length: int) -> dict[str, Any]: + if not isinstance(record, Mapping): + raise TypeError('turn_credit_invalid') + raw_pre_state = record.get('pre_state_key') + raw_post_state = record.get('post_state_key') + if raw_pre_state is None or raw_post_state is None: + raise ValueError('state_key_missing') + if not isinstance(raw_pre_state, str) or not isinstance(raw_post_state, str): + raise ValueError('state_key_invalid') + pre_state = raw_pre_state.strip() + post_state = raw_post_state.strip() + if not pre_state or not post_state: + raise ValueError('state_key_missing') + try: + start = int(record.get('action_token_start', -1)) + end = int(record.get('action_token_end', -1)) + except (TypeError, ValueError) as exc: + raise ValueError('action_span_invalid') from exc + if start < 0 or end <= start or end > sequence_length: + raise ValueError('action_span_invalid') + observation_start = record.get('observation_token_start') + observation_end = record.get('observation_token_end') + if (observation_start is None) != (observation_end is None): + raise ValueError('observation_span_invalid') + if observation_start is not None: + try: + observation_start = int(observation_start) + observation_end = int(observation_end) + except (TypeError, ValueError) as exc: + raise ValueError('observation_span_invalid') from exc + if (observation_start < end or observation_end <= observation_start or observation_end > sequence_length): + raise ValueError('observation_span_invalid') + policy_version = record.get('policy_version') + if policy_version is not None and (isinstance(policy_version, bool) + or not isinstance(policy_version, (int, str))): + raise ValueError('policy_version_invalid') + terminal_success = record.get('terminal_success') + if terminal_success is not None and not isinstance(terminal_success, bool): + raise ValueError('terminal_success_invalid') + parsed = dict(record) + parsed.update( + action_token_start=start, + action_token_end=end, + observation_token_start=observation_start, + observation_token_end=observation_end, + pre_state_key=pre_state, + post_state_key=post_state, + policy_version=policy_version, + terminal_success=terminal_success, + ) + return parsed + + def _distances( + self, + records: Sequence[tuple[int, Mapping[str, Any]]], + ) -> tuple[dict[str, int], int, int]: + reverse: dict[str, set[str]] = defaultdict(set) + nodes: set[str] = set() + edges: set[tuple[str, str]] = set() + success: set[str] = set() + for _, record in records: + pre_state, post_state = record['pre_state_key'], record['post_state_key'] + nodes.update((pre_state, post_state)) + edges.add((pre_state, post_state)) + reverse[post_state].add(pre_state) + if record.get('terminal_success') is True: + success.add(post_state) + if not success: + raise ValueError('terminal_success_missing') + if len(nodes) > self.max_graph_nodes: + raise ValueError('graph_budget_exceeded') + if len(edges) > self.max_graph_edges: + raise ValueError('graph_budget_exceeded') + distances = {node: math.inf for node in nodes} + queue: deque[str] = deque() + for node in success: + distances[node] = 0 + queue.append(node) + while queue: + child = queue.popleft() + for parent in reverse.get(child, ()): + candidate = distances[child] + 1 + if candidate < distances[parent]: + distances[parent] = candidate + queue.append(parent) + return distances, len(nodes), len(edges) + + def __call__( + self, + rewards: Sequence[float], + trajectories: Sequence[Mapping[str, Any]], + *, + num_generations: int, + sequence_lengths: Sequence[int] | None = None, + **kwargs, + ): + del kwargs + advantages, diagnostics = self.compute( + rewards, + trajectories, + num_generations=num_generations, + sequence_lengths=sequence_lengths, + ) + self.last_diagnostics = diagnostics + return advantages diff --git a/src/twinkle_agentic/envs/base.py b/src/twinkle_agentic/envs/base.py index 215137513..0c6199cdb 100644 --- a/src/twinkle_agentic/envs/base.py +++ b/src/twinkle_agentic/envs/base.py @@ -159,6 +159,15 @@ def workspace(self) -> Optional[str]: def reset(self, trajectory: Optional[Trajectory] = None) -> StepResult: return StepResult() + def state_key(self) -> Optional[str]: + """Return an opaque key for the current task state, when supported. + + The key must describe task state, not the transcript. Environments that + cannot provide a stable identifier keep the default ``None`` and are + excluded from state-conditioned credit assignment. + """ + return None + @abstractmethod def step(self, tool_name: str, arguments: Dict[str, Any]) -> StepResult: raise NotImplementedError diff --git a/src/twinkle_agentic/envs/env_tool.py b/src/twinkle_agentic/envs/env_tool.py index 5732a75b7..d38a807ad 100644 --- a/src/twinkle_agentic/envs/env_tool.py +++ b/src/twinkle_agentic/envs/env_tool.py @@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional, Tuple from twinkle.data_format.message import Tool as ToolInfo +from twinkle_agentic.tools.base import ToolExecutionResult from .base import Env, StepResult @@ -21,16 +22,26 @@ def __init__(self, self.last_result: Optional[StepResult] = None def __call__(self, tool_name: str, arguments: Dict[str, Any]) -> str: + return self.call_result(tool_name, arguments).observation + + def call_result(self, tool_name: str, arguments: Dict[str, Any]) -> ToolExecutionResult: result = self._env.step(tool_name, arguments) self.last_result = result - return result.observation + return ToolExecutionResult(observation=result.observation, info=dict(result.info or {})) def call_many(self, calls: List[Tuple[str, Dict[str, Any]]]) -> List[str]: """Batch through ``Env.step_batch``.""" + return [result.observation for result in self.call_many_results(calls)] + + def call_many_results(self, calls: List[Tuple[str, Dict[str, Any]]]) -> List[ToolExecutionResult]: + """Batch through ``Env.step_batch`` while preserving per-call info.""" results = self._env.step_batch(calls) if results: self.last_result = results[-1] - return [r.observation for r in results] + return [ToolExecutionResult(observation=r.observation, info=dict(r.info or {})) for r in results] + + def current_state_key(self) -> Optional[str]: + return self._env.state_key() def tool_info(self) -> ToolInfo: return { diff --git a/src/twinkle_agentic/envs/openenv.py b/src/twinkle_agentic/envs/openenv.py index a6a38b6d1..16e145e19 100644 --- a/src/twinkle_agentic/envs/openenv.py +++ b/src/twinkle_agentic/envs/openenv.py @@ -84,17 +84,20 @@ def _normalize_result(result) -> Dict[str, Any]: 'observation': _format_observation(result.get('observation', '')), 'reward': float(result.get('reward', 0.0)), 'done': bool(result.get('done', False)), + 'info': dict(result.get('info') or {}), } if hasattr(result, 'observation'): return { 'observation': _format_observation(result.observation), 'reward': float(getattr(result, 'reward', 0.0) or 0.0), 'done': bool(getattr(result, 'done', False)), + 'info': dict(getattr(result, 'info', {}) or {}), } return { 'observation': _format_observation(result), 'reward': float(getattr(result, 'reward', 0.0) or 0.0), 'done': bool(getattr(result, 'done', False)), + 'info': {}, } @@ -193,6 +196,7 @@ def reset(self, trajectory=None) -> StepResult: observation=normalized['observation'], reward=normalized['reward'], done=normalized['done'], + info=normalized['info'], ) def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult: @@ -208,6 +212,7 @@ def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult: observation=normalized['observation'], reward=normalized['reward'], done=normalized['done'], + info=normalized['info'], ) def close(self) -> None: @@ -434,10 +439,13 @@ def reset(self, trajectory=None) -> StepResult: # session (and its WebSocket) is reused across episodes. result = self._client.reset(**self.reset_kwargs) self._last_result = result + info = dict(getattr(result, 'info', {}) or {}) + info['episode_reward'] = self._episode_reward return StepResult( observation=_format_client_observation(getattr(result, 'observation', None)), reward=float(getattr(result, 'reward', 0.0) or 0.0), done=bool(getattr(result, 'done', False)), + info=info, ) def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult: @@ -458,11 +466,13 @@ def step(self, tool_name: str, arguments: Dict[str, Any] = None) -> StepResult: return StepResult(observation=f'Error: {e}', reward=0.0, done=False, info={'error': str(e)}) reward = float(getattr(result, 'reward', 0.0) or 0.0) self._episode_reward += reward + info = dict(getattr(result, 'info', {}) or {}) + info['episode_reward'] = self._episode_reward return StepResult( observation=_format_client_observation(getattr(result, 'observation', None)), reward=reward, done=bool(getattr(result, 'done', False)), - info={'episode_reward': self._episode_reward}, + info=info, ) def tools(self) -> List[ToolInfo]: diff --git a/src/twinkle_agentic/rollout/__init__.py b/src/twinkle_agentic/rollout/__init__.py index 20e22288e..18d13f288 100644 --- a/src/twinkle_agentic/rollout/__init__.py +++ b/src/twinkle_agentic/rollout/__init__.py @@ -1,6 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from .base import Rollout +from .credit import ToolEvent, TurnActionSpan, TurnCreditRecord from .external import ExternalRollout from .multi_turn import MultiTurnRollout -__all__ = ['ExternalRollout', 'MultiTurnRollout', 'Rollout'] +__all__ = ['ExternalRollout', 'MultiTurnRollout', 'Rollout', 'ToolEvent', 'TurnActionSpan', 'TurnCreditRecord'] diff --git a/src/twinkle_agentic/rollout/credit.py b/src/twinkle_agentic/rollout/credit.py new file mode 100644 index 000000000..ede9cd0d0 --- /dev/null +++ b/src/twinkle_agentic/rollout/credit.py @@ -0,0 +1,184 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Optional state and tool provenance for multi-turn credit assignment.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, Iterable, Mapping, Optional, Sequence, Tuple + +STATE_KEY_INFO = 'twinkle.state_key' +TERMINAL_SUCCESS_INFO = 'twinkle.terminal_success' +TOOL_EVENT_ID_INFO = 'twinkle.tool_event_id' +MAX_STATE_KEY_CHARS = 256 + +__all__ = [ + 'ObservationSpan', + 'STATE_KEY_INFO', + 'TERMINAL_SUCCESS_INFO', + 'TOOL_EVENT_ID_INFO', + 'ToolEvent', + 'TurnActionSpan', + 'TurnCreditRecord', + 'TurnCreditState', + 'make_tool_events', + 'normalize_state_key', + 'qualified_state_key', + 'state_from_tool_infos', +] + + +@dataclass(frozen=True) +class TurnActionSpan: + """Trainable action positions in the final labels index space.""" + + start: int + end: int + + +@dataclass(frozen=True) +class ObservationSpan: + """Masked positions appended between two generation turns.""" + + start: int + end: int + + +@dataclass(frozen=True) +class ToolEvent: + tool_call_id: str + tool_name: str + arguments_digest: str + result_digest: str + info: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class TurnCreditRecord: + turn_id: int + action_token_start: int + action_token_end: int + observation_token_start: int | None + observation_token_end: int | None + pre_state_key: str + post_state_key: str + tool_events: tuple[ToolEvent, ...] + policy_version: int | str | None + terminal_success: bool | None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class TurnCreditState: + available: bool + fallback_reason: str | None = None + + +def normalize_state_key(value: Any) -> str | None: + """Validate one environment-provided state key.""" + if value is None: + return None + if not isinstance(value, str): + raise TypeError(f'state_key must be a string or None, got {type(value).__name__}') + key = value.strip() + if not key: + raise ValueError('state_key must not be empty') + if len(key) > MAX_STATE_KEY_CHARS: + raise ValueError(f'state_key exceeds {MAX_STATE_KEY_CHARS} characters') + if any(ord(char) < 32 for char in key): + raise ValueError('state_key must not contain control characters') + return key + + +def qualified_state_key( + environment_id: str | None, + task_id: str | None, + local_state_key: Any, +) -> str | None: + """Build the rollout-scoped key expected by the state graph.""" + if not environment_id or not task_id: + return None + local = normalize_state_key(local_state_key) + if local is None: + return None + return f'{environment_id}:{task_id}:{local}' + + +def _digest(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha256(payload.encode('utf-8')).hexdigest() + + +def _tool_call_parts(tool_call: Any, index: int) -> tuple[str, str, Any, str]: + call = tool_call if isinstance(tool_call, Mapping) else {} + call_id = str(call.get('id') or call.get('tool_call_id') or f'tool-{index}') + function = call.get('function') if isinstance(call.get('function'), Mapping) else {} + name = str(function.get('name') or '') + arguments = function.get('arguments', {}) + if isinstance(arguments, str): + try: + decoded = json.loads(arguments) if arguments.strip() else {} + except json.JSONDecodeError: + decoded = arguments + else: + decoded = arguments + return call_id, name, decoded, _digest(decoded) + + +def make_tool_events( + tool_calls: Sequence[Any], + results: Sequence[Any], + *, + turn_id: int, +) -> tuple[ToolEvent, ...]: + """Pair tool calls with observations or named error results.""" + if len(tool_calls) != len(results): + raise ValueError('tool_calls and results must have equal length') + events = [] + for index, (call, result) in enumerate(zip(tool_calls, results)): + call_id, name, _, arguments_digest = _tool_call_parts(call, index) + if isinstance(result, Mapping): + observation = result.get('observation', '') + info = dict(result.get('info') or {}) + else: + observation = getattr(result, 'observation', result) + info = dict(getattr(result, 'info', {}) or {}) + info.setdefault(TOOL_EVENT_ID_INFO, f'turn-{turn_id}/call-{call_id}') + events.append( + ToolEvent( + tool_call_id=call_id, + tool_name=name, + arguments_digest=arguments_digest, + result_digest=_digest(str(observation)), + info=info, + )) + return tuple(events) + + +def state_from_tool_infos(infos: Iterable[Mapping[str, Any]], ) -> tuple[str | None, bool | None, str | None]: + """Return one state and terminal flag, or a conflict reason.""" + state_keys = [] + successes = [] + for info in infos: + raw_state = info.get(STATE_KEY_INFO) if isinstance(info, Mapping) else None + if raw_state is not None: + state_keys.append(normalize_state_key(raw_state)) + raw_success = info.get(TERMINAL_SUCCESS_INFO) if isinstance(info, Mapping) else None + if raw_success is not None: + if not isinstance(raw_success, bool): + raise TypeError(f'{TERMINAL_SUCCESS_INFO} must be bool when present') + successes.append(raw_success) + unique_states = set(state_keys) + unique_successes = set(successes) + if len(unique_states) > 1: + return None, None, 'state_conflict' + if len(unique_successes) > 1: + return None, None, 'terminal_success_conflict' + return ( + next(iter(unique_states)) if unique_states else None, + next(iter(unique_successes)) if unique_successes else None, + None, + ) diff --git a/src/twinkle_agentic/rollout/ledger.py b/src/twinkle_agentic/rollout/ledger.py index 053cd7be9..678c13951 100644 --- a/src/twinkle_agentic/rollout/ledger.py +++ b/src/twinkle_agentic/rollout/ledger.py @@ -32,6 +32,7 @@ from twinkle.data_format.sampling import SampledSequence from twinkle.template.base import Template from twinkle_agentic.utils.token_utils import _to_plain, append_ids, extend_with_bridge +from .credit import ObservationSpan, ToolEvent, TurnActionSpan, TurnCreditRecord, TurnCreditState class TurnLedger: @@ -55,7 +56,16 @@ class TurnLedger: which rebuilt them. """ - def __init__(self, template: Template, *, label: str = 'trajectory', max_tokens: Optional[int] = None) -> None: + def __init__( + self, + template: Template, + *, + label: str = 'trajectory', + max_tokens: Optional[int] = None, + track_turn_credits: bool = False, + initial_state_key: Optional[str] = None, + policy_version: Optional[int | str] = None, + ) -> None: """ Args: template: encodes the opening turn and every bridge after it. One @@ -70,9 +80,22 @@ def __init__(self, template: Template, *, label: str = 'trajectory', max_tokens: self.template = template self.label = label self.max_tokens = max_tokens + if policy_version is not None and (isinstance(policy_version, bool) + or not isinstance(policy_version, (int, str))): + raise TypeError('policy_version must be an int, string, or None') self._pif: Dict[str, Any] = {} self._logprobs: List[Any] = [] self._turns = 0 + self._track_turn_credits = track_turn_credits + self._initial_state_key = initial_state_key + self._current_state_key = initial_state_key + self._policy_version = policy_version + self._turn_credits: List[TurnCreditRecord] = [] + self._turn_credit_state = self._initial_credit_state() + self._pending_turn_id: Optional[int] = None + self._pending_pre_state_key: Optional[str] = None + self._pending_action_span: Optional[TurnActionSpan] = None + self._last_observation_span: Optional[ObservationSpan] = None # ---------------------------------------------------------------- reading @@ -95,12 +118,29 @@ def turns(self) -> int: """Generations banked. Not turns *attempted*: a failed call is not here.""" return self._turns + @property + def turn_credits(self) -> List[TurnCreditRecord]: + return list(self._turn_credits) + + @property + def turn_credit_state(self) -> TurnCreditState: + return self._turn_credit_state + def full(self) -> bool: """Has the episode reached :attr:`max_tokens`? Never true without one.""" if self.max_tokens is None: return False return len(self._pif.get('input_ids') or []) >= self.max_tokens + def _initial_credit_state(self) -> TurnCreditState: + if not self._track_turn_credits: + return TurnCreditState(available=False) + if self._initial_state_key is None: + return TurnCreditState(available=False, fallback_reason='state_key_missing') + if self._policy_version is None: + return TurnCreditState(available=False, fallback_reason='policy_version_missing') + return TurnCreditState(available=True) + # ---------------------------------------------------------------- writing def open(self, trajectory: Trajectory, *, tools: Optional[List[Dict[str, Any]]] = None) -> None: @@ -123,8 +163,15 @@ def open(self, trajectory: Trajectory, *, tools: Optional[List[Dict[str, Any]]] self._pif = pif self._logprobs = [] self._turns = 0 - - def record(self, seq: SampledSequence) -> None: + self._turn_credits = [] + self._current_state_key = self._initial_state_key + self._turn_credit_state = self._initial_credit_state() + self._pending_turn_id = None + self._pending_pre_state_key = None + self._pending_action_span = None + self._last_observation_span = None + + def record(self, seq: SampledSequence) -> Optional[TurnActionSpan]: """Bank one generation, taking its ids from the sampler. ``new_input_feature`` is the prompt plus what was just sampled, already @@ -138,9 +185,37 @@ def record(self, seq: SampledSequence) -> None: raise RuntimeError(f'sampler returned a SampledSequence without ' f'new_input_feature.input_ids for {self.label}; ' f'cannot continue multi-turn.') + old_ids = list(self._pif.get('input_ids') or []) self._pif = _to_plain(dict(seq.new_input_feature)) self._turns += 1 self._extend_logprobs(seq) + if not self._track_turn_credits: + return None + return self._begin_credit_turn(seq, old_ids) + + def _begin_credit_turn(self, seq: SampledSequence, old_ids: Sequence[Any]) -> Optional[TurnActionSpan]: + old_length = len(old_ids) + new_ids = list(self._pif.get('input_ids') or []) + new_length = len(self._pif.get('input_ids') or []) + labels = self._pif.get('labels') or [] + completion_mask = self._pif.get('completion_mask') + start, end = old_length - 1, new_length - 1 + valid = ( + old_length > 0 and new_length - old_length == len(seq.tokens) and new_ids[:old_length] == list(old_ids) + and new_ids[old_length:] == list(seq.tokens) and 0 <= start < end <= len(labels) + and (completion_mask is None or + (len(completion_mask) == len(labels) and all(completion_mask[pos] for pos in range(start, end))))) + if not valid: + self._turn_credit_state = TurnCreditState(False, 'action_span_invalid') + return None + span = TurnActionSpan(start=start, end=end) + self._pending_turn_id = self._turns + self._pending_pre_state_key = self._current_state_key + self._pending_action_span = span + self._last_observation_span = None + if self._pending_pre_state_key is None: + self._turn_credit_state = TurnCreditState(False, 'state_key_missing') + return span def _extend_logprobs(self, seq: SampledSequence) -> None: """Append one generation's logprobs -- one per sampled token, or raise. @@ -164,12 +239,60 @@ def observe(self, messages: Sequence[Dict[str, Any]]) -> bool: words, and training on them teaches the model to predict its own observations. False means the append did not fit. """ + before_length = len(self._pif.get('input_ids') or []) extended = extend_with_bridge(self._pif, list(messages), self.template) if extended is None: return False self._pif = extended + after_length = len(self._pif.get('input_ids') or []) + if after_length > before_length: + self._last_observation_span = ObservationSpan(start=max(0, before_length - 1), end=after_length - 1) return True + def complete_credit_turn( + self, + *, + post_state_key: Optional[str], + tool_events: Sequence[ToolEvent], + terminal_success: Optional[bool], + ) -> None: + if not self._track_turn_credits or self._pending_action_span is None: + return + if not self._turn_credit_state.available: + self.abort_credit_turn() + return + if post_state_key is None: + self._turn_credit_state = TurnCreditState(False, 'state_key_missing') + self.abort_credit_turn() + return + observation = self._last_observation_span + self._turn_credits.append( + TurnCreditRecord( + turn_id=self._pending_turn_id or self._turns, + action_token_start=self._pending_action_span.start, + action_token_end=self._pending_action_span.end, + observation_token_start=None if observation is None else observation.start, + observation_token_end=None if observation is None else observation.end, + pre_state_key=self._pending_pre_state_key or '', + post_state_key=post_state_key, + tool_events=tuple(tool_events), + policy_version=self._policy_version, + terminal_success=terminal_success, + )) + self._current_state_key = post_state_key + self._pending_turn_id = None + self._pending_pre_state_key = None + self._pending_action_span = None + self._last_observation_span = None + + def abort_credit_turn(self, reason: Optional[str] = None) -> None: + if reason and self._turn_credit_state.available: + self._turn_credit_state = TurnCreditState(False, reason) + self._pending_turn_id = None + self._pending_pre_state_key = None + self._pending_action_span = None + self._last_observation_span = None + def adopt(self, input_feature: Dict[str, Any]) -> None: """Take an already-encoded feature as the current state. @@ -245,6 +368,12 @@ def merge(self, trajectory: Trajectory, **fields: Any) -> Trajectory: out.update(self._pif) out['messages'] = list(self._pif.get('messages') or trajectory.get('messages') or []) out['logprobs'] = self._logprobs if self._logprobs else None + if self._track_turn_credits: + out['turn_credits'] = [record.to_dict() for record in self._turn_credits] + out['turn_credit_status'] = { + 'available': self._turn_credit_state.available, + 'fallback_reason': self._turn_credit_state.fallback_reason, + } out.update(fields) return out diff --git a/src/twinkle_agentic/rollout/multi_turn.py b/src/twinkle_agentic/rollout/multi_turn.py index 87c10988a..b90b482d1 100644 --- a/src/twinkle_agentic/rollout/multi_turn.py +++ b/src/twinkle_agentic/rollout/multi_turn.py @@ -9,8 +9,10 @@ from twinkle.template.base import Template from twinkle_agentic.protocol.api_sampler import APIGenerationError, APISampler from twinkle_agentic.protocol.base import API +from twinkle_agentic.tools.base import ToolExecutionResult from twinkle_agentic.tools.tool_manager import ToolManager from .base import MAX_FOLLOWUPS, STOP_GENERATION_ERROR, Rollout +from .credit import make_tool_events, qualified_state_key, state_from_tool_infos from .ledger import TurnLedger from .trace import TraceWriter @@ -179,6 +181,7 @@ def __init__( use_base_model: bool = False, stop_after_stuck_turns: int = 0, max_malformed_retries: int = 2, + track_turn_credits: bool = False, followup_fn: Optional[Callable[[Trajectory, int], Any]] = None, api: Optional[API] = None, response_callback: Optional[ResponseCallback] = None, @@ -279,6 +282,7 @@ def __init__( raise ValueError(f'max_malformed_retries must be >= 0, got ' f'{max_malformed_retries}') self.max_malformed_retries = max_malformed_retries + self.track_turn_credits = track_turn_credits # Called with (trajectory, how many follow-ups it has had already) at the # moment an episode would end: because the model stopped calling tools, # because it used up ``max_turns``, or because it was stopped for being @@ -335,6 +339,10 @@ def _resolve_call(self, kwargs: Dict[str, Any], n: int) -> Dict[str, Any]: 'tool_managers': self._broadcast(kwargs.get('tool_manager', self.tool_manager), n, name='tool_manager'), 'followup_fn': kwargs.get('followup_fn', self.followup_fn), 'harnesses': self._resolve_harness(kwargs.get('harness', self.harness), n), + 'track_turn_credits': bool(kwargs.get('track_turn_credits', self.track_turn_credits)), + 'environment_ids': self._broadcast(kwargs.get('environment_id'), n, name='environment_id'), + 'task_ids': self._broadcast(kwargs.get('task_id'), n, name='task_id'), + 'policy_versions': self._broadcast(kwargs.get('policy_version'), n, name='policy_version'), } def _resolve_harness(self, harness, n: int) -> List[Any]: @@ -362,11 +370,30 @@ def _run_episode(self, trajectory: Trajectory, index: int, ctx: Dict[str, Any], followup_fn = ctx['followup_fn'] adapter_kwargs: Dict[str, Any] = ctx['adapter_kwargs'] response_callback: ResponseCallback = ctx['response_callback'] + track_turn_credits = ctx['track_turn_credits'] + requested_environment_id = ctx['environment_ids'][index] + requested_task_id = ctx['task_ids'][index] + requested_policy_version = ctx['policy_versions'][index] + environment_id = ( + requested_environment_id if requested_environment_id is not None else trajectory.get('environment_id')) + task_id = requested_task_id if requested_task_id is not None else trajectory.get('task_id') + policy_version = ( + requested_policy_version if requested_policy_version is not None else trajectory.get('policy_version')) # The token account for this episode. Every id the trajectory ends up # trained on passes through it; what stays in this function is the policy # that decides when to add one. See ``ledger.py``. - ledger = TurnLedger(self.template, label=f'trajectory {index}', max_tokens=self.max_trajectory_tokens) + initial_state_key = ( + qualified_state_key(environment_id, task_id, self._current_state_key(tool_manager)) + if track_turn_credits else None) + ledger = TurnLedger( + self.template, + label=f'trajectory {index}', + max_tokens=self.max_trajectory_tokens, + track_turn_credits=track_turn_credits, + initial_state_key=initial_state_key, + policy_version=policy_version, + ) # A trajectory that named no tools advertises the manager's, so the prompt # lists what can actually be dispatched. # A harness may shape the opening (system prompt, tool schema) before @@ -528,6 +555,7 @@ def grant_followup() -> bool: parse_errors = ([] if followups else self.template.tool_call_errors(seq.decoded or '')) if parse_errors and malformed_turns < self.max_malformed_retries: malformed_turns += 1 + ledger.abort_credit_turn() if not ledger.observe([_malformed_tool_message(parse_errors)]): truncated = True break @@ -537,6 +565,7 @@ def grant_followup() -> bool: # why this is not a second rollout. if grant_followup(): continue + ledger.abort_credit_turn() break if ledger.turns >= self.max_turns: @@ -554,7 +583,30 @@ def grant_followup() -> bool: # messages the model did not write. if tool_manager is None: raise ValueError('the model emitted tool_calls but this trajectory has no ToolManager') - observations = self._run_tools(tool_manager, tool_calls) + tool_results = self._run_tools_results(tool_manager, tool_calls) + observations = [result.observation for result in tool_results] + terminal_success = None + state_error = None + post_state_key = None + if track_turn_credits: + try: + local_post_state, terminal_success, state_error = state_from_tool_infos( + [result.info for result in tool_results]) + except (TypeError, ValueError): + local_post_state, terminal_success, state_error = None, None, 'state_key_invalid' + current_state = self._current_state_key(tool_manager) + explicit_post_state = qualified_state_key(environment_id, task_id, local_post_state) + current_post_state = qualified_state_key(environment_id, task_id, current_state) + if explicit_post_state is not None and current_post_state is not None: + post_state_key = explicit_post_state + if explicit_post_state != current_post_state: + state_error = 'state_conflict' + else: + post_state_key = explicit_post_state or current_post_state + if state_error is not None: + ledger.abort_credit_turn(state_error) + elif post_state_key is None: + ledger.abort_credit_turn('state_key_missing') if self.stop_after_stuck_turns: keys = [_call_key(tc) for tc in tool_calls] all_repeats = bool(keys) and all(k in seen_calls for k in keys) @@ -570,7 +622,19 @@ def grant_followup() -> bool: else: tool_messages = _default_tool_messages(tool_calls, observations) overflowed = not ledger.observe(tool_messages) + if track_turn_credits and not overflowed and state_error is None and post_state_key is not None: + try: + tool_events = make_tool_events(tool_calls, tool_results, turn_id=ledger.turns) + except (TypeError, ValueError): + ledger.abort_credit_turn('tool_event_invalid') + else: + ledger.complete_credit_turn( + post_state_key=post_state_key, + tool_events=tool_events, + terminal_success=terminal_success, + ) if overflowed: + ledger.abort_credit_turn('observation_overflow') # Trajectory exceeded max_length. truncated = True else: @@ -640,6 +704,29 @@ def _harness_tool_messages(harness, pif: Dict[str, Any], observations, tool_call tail = shaped_msgs[len(prior):] return tail or _default_tool_messages(tool_calls, observations) + @staticmethod + def _current_state_key(tool_manager: Optional[ToolManager]) -> Optional[str]: + return None if tool_manager is None else tool_manager.current_state_key() + + @staticmethod + def _run_tools_results( + tool_manager: ToolManager, + tool_calls: List[Dict[str, Any]], + ) -> List[ToolExecutionResult]: + if hasattr(tool_manager, 'call_many_results'): + results = list(tool_manager.call_many_results(tool_calls)) + else: + contents = tool_manager.call_many(tool_calls) if hasattr( + tool_manager, 'call_many') else [tool_manager(tool_call) for tool_call in tool_calls] + results = [ToolExecutionResult(observation='' if content is None else str(content)) for content in contents] + if len(results) < len(tool_calls): + failure = ToolExecutionResult( + observation=f'the tool manager returned {len(results)} results for {len(tool_calls)} calls', + info={'error': 'tool_result_count_mismatch'}, + ) + results.extend([failure] * (len(tool_calls) - len(results))) + return results[:len(tool_calls)] + @staticmethod def _run_tools(tool_manager: ToolManager, tool_calls: List[Dict[str, Any]]) -> List[str]: """Run one turn's calls, through ``call_many`` when the manager has it. @@ -652,11 +739,4 @@ def _run_tools(tool_manager: ToolManager, tool_calls: List[Dict[str, Any]]) -> L A manager that answers with fewer results than calls leaves the rest empty rather than shifting them onto the wrong call. """ - if hasattr(tool_manager, 'call_many'): - contents = tool_manager.call_many(tool_calls) - else: - contents = [tool_manager(tc) for tc in tool_calls] - obs = [''] * len(tool_calls) - for i, content in enumerate(contents[:len(tool_calls)]): - obs[i] = '' if content is None else str(content) - return obs + return [result.observation for result in MultiTurnRollout._run_tools_results(tool_manager, tool_calls)] diff --git a/src/twinkle_agentic/tools/base.py b/src/twinkle_agentic/tools/base.py index 92496a695..1fa4a5769 100644 --- a/src/twinkle_agentic/tools/base.py +++ b/src/twinkle_agentic/tools/base.py @@ -1,10 +1,19 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from abc import ABC, abstractmethod +from dataclasses import dataclass, field from typing import Any, Dict from twinkle.data_format import Tool as ToolInfo +@dataclass(frozen=True) +class ToolExecutionResult: + """One tool call's observation and structured provenance.""" + + observation: str = '' + info: Dict[str, Any] = field(default_factory=dict) + + class Tool(ABC): @abstractmethod diff --git a/src/twinkle_agentic/tools/tool_manager.py b/src/twinkle_agentic/tools/tool_manager.py index 2bdf01410..fea373af6 100644 --- a/src/twinkle_agentic/tools/tool_manager.py +++ b/src/twinkle_agentic/tools/tool_manager.py @@ -6,7 +6,7 @@ from twinkle.data_format import ToolCall from twinkle.data_format.message import Tool as ToolInfo from twinkle.utils import get_logger -from twinkle_agentic.tools.base import Tool +from twinkle_agentic.tools.base import Tool, ToolExecutionResult logger = get_logger() @@ -122,26 +122,41 @@ def tool_infos(self) -> List[ToolInfo]: return [t.tool_info() for t in self._tools.values()] def __call__(self, tool_call: Union[ToolCall, Dict[str, Any]]) -> str: + return self.call_result(tool_call).observation + + def call_result(self, tool_call: Union[ToolCall, Dict[str, Any]]) -> ToolExecutionResult: name, args, err = _unpack_tool_call(tool_call) if err: - return err + return ToolExecutionResult(observation=err, info={'error': err}) if (tool := self._tools.get(name)) is None: available = ', '.join(sorted(self._tools)) or '(none)' hint = '' if (suggestion := _suggest(name, self._tools)) is not None: hint = (f' Did you mean {suggestion!r}? Tool names must be given in ' f'full, including the part before "---".') - return f'Error: unknown tool {name!r}.{hint} Available: {available}.' + error = f'Error: unknown tool {name!r}.{hint} Available: {available}.' + return ToolExecutionResult(observation=error, info={'error': error}) try: - return str(tool(name, args)) + if hasattr(tool, 'call_result'): + result = tool.call_result(name, args) + return ToolExecutionResult(observation=str(result.observation), info=dict(result.info or {})) + return ToolExecutionResult(observation=str(tool(name, args))) except Exception as e: # noqa - return f'Error: tool {name!r} raised {type(e).__name__}: {e}' + error = f'Error: tool {name!r} raised {type(e).__name__}: {e}' + return ToolExecutionResult(observation=error, info={'error': error}) def call_many( self, tool_calls: Iterable[Union[ToolCall, Dict[str, Any]]], max_workers: Optional[int] = None, ) -> List[str]: + return [result.observation for result in self.call_many_results(tool_calls, max_workers=max_workers)] + + def call_many_results( + self, + tool_calls: Iterable[Union[ToolCall, Dict[str, Any]]], + max_workers: Optional[int] = None, + ) -> List[ToolExecutionResult]: """Execute many tool calls, preserving input order. ``tool_calls`` are the OpenAI-shaped dicts produced by @@ -169,18 +184,18 @@ def call_many( if not calls: return [] if len(calls) == 1: - return [self(calls[0])] + return [self.call_result(calls[0])] unpacked = [_unpack_tool_call(tc) for tc in calls] env = self._shared_env() if env is not None: - out: List[Optional[str]] = [None] * len(calls) + out: List[Optional[ToolExecutionResult]] = [None] * len(calls) batched: List[Tuple[int, str, Dict[str, Any]]] = [] for i, (name, args, err) in enumerate(unpacked): if err is None and name in self._tools: batched.append((i, name, args)) else: - out[i] = self(calls[i]) + out[i] = self.call_result(calls[i]) try: results = env.step_batch([(name, args) for _i, name, args in batched]) except Exception as e: # noqa @@ -194,8 +209,8 @@ def call_many( f'{type(e).__name__}: {e}') failure = f'Error: tool batch did not run: {type(e).__name__}: {e}' for i, _name, _args in batched: - out[i] = failure - return ['' if x is None else x for x in out] + out[i] = ToolExecutionResult(observation=failure, info={'error': failure}) + return [ToolExecutionResult() if x is None else x for x in out] if len(results) != len(batched): # Same reasoning: a short result list means the calls did not all # run, and pairing them up by position would report one call's @@ -205,19 +220,27 @@ def call_many( failure = (f'Error: tool batch did not run: the environment returned ' f'{len(results)} results for {len(batched)} calls.') for i, _name, _args in batched: - out[i] = failure - return ['' if x is None else x for x in out] + out[i] = ToolExecutionResult(observation=failure, info={'error': failure}) + return [ToolExecutionResult() if x is None else x for x in out] for (i, _name, _args), r in zip(batched, results): - out[i] = r.observation if hasattr(r, 'observation') else str(r) - return ['' if x is None else x for x in out] + observation = r.observation if hasattr(r, 'observation') else str(r) + info = dict(getattr(r, 'info', {}) or {}) + out[i] = ToolExecutionResult(observation=observation, info=info) + return [ToolExecutionResult() if x is None else x for x in out] workers = max_workers or min(32, len(calls)) - out = [None] * len(calls) + out: List[Optional[ToolExecutionResult]] = [None] * len(calls) with ThreadPoolExecutor(max_workers=workers) as pool: - futs = {pool.submit(self, tc): i for i, tc in enumerate(calls)} + futs = {pool.submit(self.call_result, tc): i for i, tc in enumerate(calls)} for fut in as_completed(futs): out[futs[fut]] = fut.result() - return ['' if x is None else x for x in out] + return [ToolExecutionResult() if x is None else x for x in out] + + def current_state_key(self) -> Optional[str]: + env = self._shared_env() + if env is None: + return None + return env.state_key() def _shared_env(self): """Return the Env shared by every registered EnvTool, else None.""" diff --git a/tests/advantage/test_graphcredit_gate1.py b/tests/advantage/test_graphcredit_gate1.py new file mode 100644 index 000000000..667486388 --- /dev/null +++ b/tests/advantage/test_graphcredit_gate1.py @@ -0,0 +1,142 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import importlib.util +import sys +from types import SimpleNamespace +from pathlib import Path + +import torch +import pytest + +from twinkle.advantage import StateGraphAdvantage + + +MODULE_PATH = Path(__file__).resolve().parents[2] / 'cookbook/rl/graphcredit/gate1.py' +SPEC = importlib.util.spec_from_file_location('graphcredit_gate1', MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +gate1 = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = gate1 +SPEC.loader.exec_module(gate1) + + +def test_grid_task_and_parse_action(): + task = gate1.make_task(7, grid_size=5, distance=2) + assert abs(task.start[0] - task.goal[0]) + abs(task.start[1] - task.goal[1]) == 2 + assert gate1.parse_action(' RIGHT ') == 'right' + assert gate1.parse_action('right') == 'right' + assert gate1.parse_action('I choose a wall') is None + + +def test_wrong_move_ends_episode_without_progress(): + task = gate1.GridTask( + task_uid=1, + grid_size=5, + distance=2, + start=(2, 2), + goal=(2, 4), + ) + episode = gate1.Episode(task=task, uid='bad', position=task.start) + + gate1.step_episode(episode, 'left') + + assert episode.done is True + assert episode.success is False + assert episode.reward == 0.0 + + +def _episode(task, actions): + episode = gate1.Episode(task=task, uid='episode', position=task.start) + for turn_id, action in enumerate(actions): + pre_state = task.state_key(episode.position) + start = 4 + turn_id * 4 + gate1.step_episode(episode, action) + post_state = ( + task.state_key(episode.position) + if episode.success or not episode.done else task.dead_state_key + ) + episode.turns.append( + gate1.EpisodeTurn( + prompt='', + action=action, + pre_state_key=pre_state, + post_state_key=post_state, + action_token_start=start, + action_token_end=start + 2, + terminal_success=True if episode.success else False, + )) + if episode.done: + break + episode.input_ids = list(range(4 + len(episode.turns) * 4 + 2)) + return episode + + +def test_graph_credit_prefers_useful_intermediate_action(): + task = gate1.GridTask( + task_uid=2, + grid_size=5, + distance=2, + start=(2, 2), + goal=(2, 4), + ) + success = _episode(task, ['right', 'right']) + failure = _episode(task, ['left']) + trajectories = [ + success.to_trajectory(policy_version=1), + failure.to_trajectory(policy_version=1), + ] + + advantage = StateGraphAdvantage(beta_episode=0.0, beta_graph=1.0) + result = advantage([1.0, 0.0], trajectories, num_generations=2) + + success_start = success.turns[0].action_token_start + failure_start = failure.turns[0].action_token_start + assert torch.all(result[0, success_start:success_start + 2] > 0) + assert torch.all(result[1, failure_start:failure_start + 2] < 0) + + +def test_train_counts_completed_updates_but_not_skipped_attempts(monkeypatch): + task = gate1.GridTask( + task_uid=3, + grid_size=5, + distance=2, + start=(2, 2), + goal=(2, 4), + ) + episode = _episode(task, ['right', 'right']) + sampled = (task, [episode]) + samples = iter([None, sampled, None, sampled]) + policy_versions = [] + + monkeypatch.setattr(gate1, 'evaluate', lambda *args, **kwargs: { + 'success_rate': 0.0, + 'invalid_rate': 0.0, + }) + monkeypatch.setattr(gate1, 'reset_peak_hbm', lambda: None) + monkeypatch.setattr(gate1, 'synchronize', lambda device: None) + monkeypatch.setattr(gate1, 'peak_hbm_gb', lambda: 0.0) + monkeypatch.setattr(gate1, 'sample_training_group', lambda *args, **kwargs: next(samples)) + + def compute_advantages(mode, episodes, policy_version, group_size, args): + policy_versions.append(policy_version) + return torch.zeros(1), {'fallback_counts': {}} + + monkeypatch.setattr(gate1, 'compute_advantages', compute_advantages) + monkeypatch.setattr(gate1, 'policy_gradient_update', lambda *args, **kwargs: (0.0, 1)) + args = SimpleNamespace( + learning_rate=5e-5, + steps=2, + max_step_attempts=3, + eval_every=0, + seed=20261119, + mode='graphcredit', + group_size=2, + max_grad_norm=1.0, + ) + tokenizer = SimpleNamespace(pad_token_id=0) + + result = gate1.train(torch.nn.Linear(1, 1), tokenizer, args, torch.device('cpu')) + + assert result['completed_steps'] == 2 + assert result['attempted_steps'] == 4 + assert result['skipped_steps'] == 2 + assert policy_versions == [1, 2] + assert [record['step'] for record in result['history'] if not record.get('skipped')] == [1, 2] diff --git a/tests/advantage/test_state_graph_advantage.py b/tests/advantage/test_state_graph_advantage.py new file mode 100644 index 000000000..cc674715f --- /dev/null +++ b/tests/advantage/test_state_graph_advantage.py @@ -0,0 +1,161 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import pytest +import torch + +from twinkle.advantage import GRPOAdvantage, StateGraphAdvantage + + +def _trajectory(pre_state, post_state, success, *, version=1, span=(1, 2)): + return { + 'labels': [0] * 6, + 'turn_credit_status': { + 'available': True, + 'fallback_reason': None, + }, + 'turn_credits': [{ + 'turn_id': 1, + 'action_token_start': span[0], + 'action_token_end': span[1], + 'pre_state_key': pre_state, + 'post_state_key': post_state, + 'policy_version': version, + 'terminal_success': success, + }], + } + + +class TestStateGraphAdvantage: + + def test_adds_state_relative_credit_to_action_tokens(self): + trajectories = [ + _trajectory('s0', 'success', True), + _trajectory('s0', 'dead', None), + _trajectory('s1', 'success', True), + _trajectory('s1', 'dead', None), + ] + advantage = StateGraphAdvantage() + result = advantage([1.0, 0.0, 1.0, 0.0], trajectories, num_generations=4) + + episode = GRPOAdvantage()(torch.tensor([1.0, 0.0, 1.0, 0.0]), num_generations=4) + assert torch.allclose(result[:, 0], episode) + assert torch.all(result[0, 1] > episode[0]) + assert torch.all(result[1, 1] < episode[1]) + assert advantage.last_diagnostics.comparable_state_group_count == 2 + assert advantage.last_diagnostics.transition_count == 4 + assert advantage.last_diagnostics.fallback_counts == {} + + def test_missing_success_is_exact_grpo_fallback(self): + trajectories = [ + _trajectory('s0', 'dead0', None), + _trajectory('s0', 'dead1', None), + _trajectory('s0', 'dead2', None), + _trajectory('s0', 'dead3', None), + ] + rewards = [0.0, 1.0, 0.0, 2.0] + advantage = StateGraphAdvantage() + result = advantage(rewards, trajectories, num_generations=4) + episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=4) + + assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6)) + assert advantage.last_diagnostics.fallback_counts['terminal_success_missing'] == 1 + + def test_policy_version_mismatch_is_exact_grpo_fallback(self): + trajectories = [ + _trajectory('s0', 'success', True, version=1), + _trajectory('s0', 'dead', None, version=2), + ] + rewards = [1.0, 0.0] + advantage = StateGraphAdvantage() + result = advantage(rewards, trajectories, num_generations=2) + episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2) + + assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6)) + assert advantage.last_diagnostics.fallback_counts['policy_version_mismatch'] == 1 + + def test_partially_missing_policy_version_is_exact_grpo_fallback(self): + trajectories = [ + _trajectory('s0', 'success', True, version=1), + _trajectory('s0', 'dead', None, version=None), + ] + rewards = [1.0, 0.0] + advantage = StateGraphAdvantage() + result = advantage(rewards, trajectories, num_generations=2) + episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2) + + assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6)) + assert advantage.last_diagnostics.fallback_counts['policy_version_missing'] == 1 + + def test_fully_missing_policy_version_is_exact_grpo_fallback(self): + trajectories = [ + _trajectory('s0', 'success', True, version=None), + _trajectory('s0', 'dead', None, version=None), + ] + rewards = [1.0, 0.0] + advantage = StateGraphAdvantage() + result = advantage(rewards, trajectories, num_generations=2) + episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2) + + assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6)) + assert advantage.last_diagnostics.fallback_counts['policy_version_missing'] == 1 + + def test_policy_version_zero_is_available(self): + trajectories = [ + _trajectory('s0', 'success', True, version=0), + _trajectory('s0', 'dead', None, version=0), + ] + advantage = StateGraphAdvantage() + result = advantage([1.0, 0.0], trajectories, num_generations=2) + + assert result.shape == (2, 6) + assert advantage.last_diagnostics.fallback_counts == {} + + @pytest.mark.parametrize('field,value,reason', [ + ('pre_state_key', 1, 'state_key_invalid'), + ('policy_version', [], 'policy_version_invalid'), + ('terminal_success', 'yes', 'terminal_success_invalid'), + ('observation_token_start', 1, 'observation_span_invalid'), + ]) + def test_invalid_contract_field_is_exact_grpo_fallback(self, field, value, reason): + trajectories = [_trajectory('s0', 'success', True), _trajectory('s0', 'dead', None)] + trajectories[0]['turn_credits'][0][field] = value + rewards = [1.0, 0.0] + advantage = StateGraphAdvantage() + result = advantage(rewards, trajectories, num_generations=2) + episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2) + + assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6)) + assert advantage.last_diagnostics.fallback_counts[reason] == 1 + + def test_insufficient_state_support_leaves_only_episode_credit(self): + trajectories = [ + _trajectory('s0', 'success', True), + _trajectory('s1', 'dead', None), + ] + rewards = [1.0, 0.0] + advantage = StateGraphAdvantage() + result = advantage(rewards, trajectories, num_generations=2) + episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2) + + assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6)) + assert advantage.last_diagnostics.comparable_state_group_count == 0 + + def test_invalid_span_skips_that_row(self): + trajectories = [ + _trajectory('s0', 'success', True, span=(0, 1)), + _trajectory('s0', 'dead', None, span=(7, 9)), + ] + rewards = [1.0, 0.0] + advantage = StateGraphAdvantage() + result = advantage(rewards, trajectories, num_generations=2) + episode = GRPOAdvantage()(torch.tensor(rewards), num_generations=2) + + assert torch.allclose(result, episode.unsqueeze(1).repeat(1, 6)) + assert advantage.last_diagnostics.fallback_counts['action_span_invalid'] == 1 + + def test_validation(self): + with pytest.raises(ValueError, match='at least two'): + StateGraphAdvantage()([1.0], [_trajectory('s', 't', True)], num_generations=1) + with pytest.raises(ValueError, match='complete groups'): + StateGraphAdvantage()([1.0], [_trajectory('s', 't', True)], num_generations=2) + with pytest.raises(ValueError, match='min_state_support'): + StateGraphAdvantage(min_state_support=1) diff --git a/tests/twinkle_agentic/test_multi_turn_turn_credit.py b/tests/twinkle_agentic/test_multi_turn_turn_credit.py new file mode 100644 index 000000000..94a47e7ec --- /dev/null +++ b/tests/twinkle_agentic/test_multi_turn_turn_credit.py @@ -0,0 +1,110 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import json +import sys +import types + +try: + import resource # noqa: F401 +except ModuleNotFoundError: + stub = types.ModuleType('resource') + stub.RLIMIT_AS = 0 + stub.setrlimit = lambda *_args, **_kwargs: None + sys.modules['resource'] = stub + +from twinkle_agentic.envs.base import Env, StepResult +from twinkle_agentic.envs.env_tool import EnvTool +from twinkle_agentic.rollout.credit import STATE_KEY_INFO, TERMINAL_SUCCESS_INFO +from twinkle_agentic.rollout.multi_turn import MultiTurnRollout +from twinkle_agentic.tools.tool_manager import ToolManager +from tests.twinkle_agentic.test_multi_turn_rollout import FakeSampler, FakeTemplate, FakeTokenizer + + +class StatefulEnv(Env): + + def __init__(self): + self.state = 's0' + + def state_key(self): + return self.state + + def step(self, tool_name, arguments): + self.state = arguments.get('next_state', self.state) + return StepResult( + observation=f'state={self.state}', + info={ + STATE_KEY_INFO: self.state, + TERMINAL_SUCCESS_INFO: arguments.get('success'), + }, + ) + + +def _rollout(sampler, template, tool_manager, *, track): + return MultiTurnRollout( + sampler=sampler, + template=template, + tool_manager=tool_manager, + max_turns=3, + track_turn_credits=track, + ) + + +def test_multi_turn_emits_state_key_and_action_span(): + tokenizer = FakeTokenizer() + template = FakeTemplate(tokenizer) + sampler = FakeSampler(template) + env = StatefulEnv() + tool_manager = ToolManager(EnvTool.from_schemas(env, [{ + 'type': 'function', + 'function': { + 'name': 'act' + } + }])) + sampler.queue( + '' + + json.dumps({ + 'name': 'act', + 'arguments': { + 'next_state': 's1', + 'success': True, + }, + }) + + '') + sampler.queue('done') + rollout = _rollout(sampler, template, tool_manager, track=True) + trajectory = rollout( + [{ + 'messages': [{ + 'role': 'user', + 'content': 'go' + }], + }], + environment_id='env', + task_id='task', + policy_version=7, + )[0] + + assert trajectory['turn_credit_status']['available'] is True + assert len(trajectory['turn_credits']) == 1 + credit = trajectory['turn_credits'][0] + assert credit['pre_state_key'] == 'env:task:s0' + assert credit['post_state_key'] == 'env:task:s1' + assert credit['action_token_start'] < credit['action_token_end'] + assert credit['policy_version'] == 7 + assert credit['terminal_success'] is True + assert credit['tool_events'][0]['tool_name'] == 'act' + + +def test_multi_turn_keeps_turn_credit_fields_absent_by_default(): + tokenizer = FakeTokenizer() + template = FakeTemplate(tokenizer) + sampler = FakeSampler(template) + sampler.queue('done') + trajectory = _rollout(sampler, template, None, track=False)( + [{ + 'messages': [{ + 'role': 'user', + 'content': 'go' + }], + }])[0] + assert 'turn_credits' not in trajectory + assert 'turn_credit_status' not in trajectory diff --git a/tests/twinkle_agentic/test_turn_credit.py b/tests/twinkle_agentic/test_turn_credit.py new file mode 100644 index 000000000..0b3a943be --- /dev/null +++ b/tests/twinkle_agentic/test_turn_credit.py @@ -0,0 +1,219 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import sys +import types + +try: + import resource # noqa: F401 +except ModuleNotFoundError: + stub = types.ModuleType('resource') + stub.RLIMIT_AS = 0 + stub.setrlimit = lambda *_args, **_kwargs: None + sys.modules['resource'] = stub + +from twinkle.data_format.sampling import SampledSequence +from twinkle_agentic.envs.base import Env, StepResult +from twinkle_agentic.envs.env_tool import EnvTool +from twinkle_agentic.rollout.credit import ( + STATE_KEY_INFO, + TERMINAL_SUCCESS_INFO, + make_tool_events, + qualified_state_key, + state_from_tool_infos, +) +from twinkle_agentic.rollout.ledger import TurnLedger +from twinkle_agentic.tools.tool_manager import ToolManager + + +class StatefulEnv(Env): + + def __init__(self): + self.state = 's0' + + def state_key(self): + return self.state + + def step(self, tool_name, arguments): + return self.step_batch([(tool_name, arguments)])[0] + + def step_batch(self, calls): + results = [] + for _, arguments in calls: + self.state = arguments.get('next_state', self.state) + results.append( + StepResult( + observation=f'reached {self.state}', + info={ + STATE_KEY_INFO: self.state, + TERMINAL_SUCCESS_INFO: arguments.get('success'), + }, + )) + return results + + +class FakeTemplate: + + def encode(self, trajectory, add_generation_prompt=False): + del add_generation_prompt + messages = trajectory.get('messages') or [] + ids = list(range(len(messages) + 2)) + return { + 'messages': messages, + 'input_ids': ids, + 'labels': list(ids), + 'completion_mask': [0] * len(ids), + } + + +def test_tool_manager_preserves_info_without_changing_call_many(): + env = StatefulEnv() + manager = ToolManager(EnvTool.from_schemas(env, [{ + 'type': 'function', + 'function': { + 'name': 'act' + } + }])) + calls = [{ + 'id': 'c1', + 'type': 'function', + 'function': { + 'name': 'act', + 'arguments': { + 'next_state': 's1' + } + }, + }] + + rich = manager.call_many_results(calls) + assert len(rich) == 1 + assert rich[0].observation == 'reached s1' + assert rich[0].info[STATE_KEY_INFO] == 's1' + assert manager.call_many(calls) == ['reached s1'] + assert manager.current_state_key() == 's1' + + +def test_tool_manager_marks_unknown_calls_without_losing_metadata(): + manager = ToolManager() + result = manager.call_result({ + 'id': 'missing', + 'function': { + 'name': 'not_registered', + 'arguments': {} + } + }) + assert 'unknown tool' in result.observation + assert 'error' in result.info + + +def test_state_and_tool_provenance_helpers(): + assert qualified_state_key('env', 'task', ' s1 ') == 'env:task:s1' + assert qualified_state_key(None, 'task', 's1') is None + assert state_from_tool_infos([ + {STATE_KEY_INFO: 's1'}, + {STATE_KEY_INFO: 's1', TERMINAL_SUCCESS_INFO: True}, + ]) == ('s1', True, None) + assert state_from_tool_infos([ + {STATE_KEY_INFO: 's1'}, + {STATE_KEY_INFO: 's2'}, + ]) == (None, None, 'state_conflict') + + events = make_tool_events( + [{ + 'id': 'a', + 'function': { + 'name': 'act', + 'arguments': {'next_state': 's1'} + } + }], + [StepResult(observation='ok', info={STATE_KEY_INFO: 's1'})], + turn_id=1, + ) + assert events[0].tool_call_id == 'a' + assert events[0].tool_name == 'act' + assert len(events[0].arguments_digest) == 64 + assert len(events[0].result_digest) == 64 + + +def test_turn_ledger_records_action_span_and_state_edge(): + ledger = TurnLedger( + FakeTemplate(), + track_turn_credits=True, + initial_state_key='env:task:s0', + policy_version=3, + ) + ledger.open({'messages': [{'role': 'user', 'content': 'go'}]}) + old_length = len(ledger.input_feature['input_ids']) + new_feature = { + **ledger.input_feature, + 'input_ids': [0, 1, 2, 10, 11, 12], + 'labels': [0, 1, 10, 11, 12, -100], + 'completion_mask': [0, 0, 1, 1, 1, 0], + } + span = ledger.record(SampledSequence( + stop_reason='stop', + tokens=[10, 11, 12], + logprobs=None, + new_input_feature=new_feature, + )) + assert span.start == old_length - 1 + assert span.end == len(new_feature['input_ids']) - 1 + ledger.complete_credit_turn( + post_state_key='env:task:s1', + tool_events=(), + terminal_success=True, + ) + trajectory = ledger.merge({'messages': []}) + assert trajectory['turn_credit_status']['available'] is True + record = trajectory['turn_credits'][0] + assert record['pre_state_key'] == 'env:task:s0' + assert record['post_state_key'] == 'env:task:s1' + assert record['action_token_start'] == span.start + assert record['action_token_end'] == span.end + assert record['policy_version'] == 3 + + +def test_turn_ledger_marks_missing_initial_state_as_fallback(): + ledger = TurnLedger(FakeTemplate(), track_turn_credits=True) + trajectory = ledger.merge({'messages': []}) + assert trajectory['turn_credit_status'] == { + 'available': False, + 'fallback_reason': 'state_key_missing', + } + + +def test_turn_ledger_requires_policy_version_when_tracking(): + ledger = TurnLedger(FakeTemplate(), track_turn_credits=True, initial_state_key='env:task:s0') + trajectory = ledger.merge({'messages': []}) + assert trajectory['turn_credit_status'] == { + 'available': False, + 'fallback_reason': 'policy_version_missing', + } + + +def test_turn_ledger_rejects_non_sampled_tail(): + ledger = TurnLedger( + FakeTemplate(), + track_turn_credits=True, + initial_state_key='env:task:s0', + policy_version=0, + ) + ledger.open({'messages': [{'role': 'user', 'content': 'go'}]}) + old_feature = ledger.input_feature + new_feature = { + **old_feature, + 'input_ids': [0, 1, 2, 10, 11, 99], + 'labels': [0, 1, 10, 11, 99, -100], + 'completion_mask': [0, 0, 1, 1, 1, 0], + } + span = ledger.record(SampledSequence( + stop_reason='stop', + tokens=[10, 11, 12], + logprobs=None, + new_input_feature=new_feature, + )) + trajectory = ledger.merge({'messages': []}) + + assert span is None + assert trajectory['turn_credit_status'] == { + 'available': False, + 'fallback_reason': 'action_span_invalid', + }