Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
14ee9bd
fix(model): preserve task training and checkpoint state
Yunnglin Nov 14, 2026
a3ad125
fix(server): preserve family loaders and task data contracts
Yunnglin Nov 14, 2026
42bdd0b
fix(server): close task checkpoint and model admission gaps
Yunnglin Nov 16, 2026
8179ab1
fix(megatron): apply deterministic kernels before multi-LoRA wrapping
Yunnglin Nov 16, 2026
3e1c34c
test(megatron): isolate eager-wrap constructor regression
Yunnglin Nov 16, 2026
4817a8c
fix(fsdp): wrap task head slots before tenants bind
Yunnglin Nov 16, 2026
3711c87
test: remove kernel helper checks covered by model construction
Yunnglin Nov 16, 2026
df75adf
docs: keep training guides focused on user actions
Yunnglin Nov 16, 2026
9b9a490
docs: remove incomplete decision model setup section
Yunnglin Nov 16, 2026
87f7f2b
fix(model): harden checkpoint resume contracts
Yunnglin Nov 16, 2026
2ab4552
fix(fsdp): skip empty optimizer state on non-master ranks
Yunnglin Nov 17, 2026
9ee344c
fix(lora): preserve multimodal targets across tenant slots
Yunnglin Nov 17, 2026
aa1c496
refactor: simplify checkpoint and response handling
Yunnglin Nov 17, 2026
c1e63aa
refactor: consolidate backend validation and task-head sharding
Yunnglin Nov 17, 2026
83a9ee8
fix(fsdp): restore sharded task heads without extra BF16 groups
Yunnglin Nov 17, 2026
4b25dc4
refactor: remove experiment-only deterministic kernel overrides
Yunnglin Nov 17, 2026
1cccc57
refactor: share sharded tensor copies for task checkpoints
Yunnglin Nov 17, 2026
bdfdfba
refactor(server): keep model file validation at worker boundary
Yunnglin Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/source_en/Usage Guide/Train-as-a-Service.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ for epoch in range(2):
print(f'Saved checkpoint for epoch {epoch} to {result.path}')
```

On the Transformers backend, finish the current gradient accumulation, optimizer update, and gradient clearing (`zero_grad`) before saving a training checkpoint with Tinker's `save_state()` or Twinkle's `save(..., save_optimizer=True)`. Weights-only exports do not have this restriction.

With the code above, you can train a self-cognition LoRA based on `Qwen/Qwen3.8-27B`. This LoRA will change the model's name and creator to the names specified during training. To perform inference using this LoRA:

```python
Expand Down
2 changes: 2 additions & 0 deletions docs/source_zh/使用指引/训练服务.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ for epoch in range(2):
print(f'Saved checkpoint for epoch {epoch} to {result.path}')
```

使用 Transformers 后端时,请先完成当前梯度累积、优化器更新和梯度清理(`zero_grad`),再调用 Tinker 的 `save_state()` 或 Twinkle 的 `save(..., save_optimizer=True)` 保存训练断点。仅导出模型权重不受此限制。

通过上述代码,你可以训练一个原模型为`Qwen/Qwen3.8-27B`的自我认知lora。这个lora会改变模型的名称和制造者为训练时指定的名称。使用这个lora进行推理:

```python
Expand Down
27 changes: 16 additions & 11 deletions src/twinkle/model/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@
import shutil
from abc import ABC, abstractmethod
from concurrent.futures import Future, ThreadPoolExecutor
from itertools import chain
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Protocol, Type, Union

from twinkle import Platform, remote_function
from twinkle import Platform, remote_function, torch_util
from twinkle.data_format import InputFeature, ModelOutput
from twinkle.hub import HubOperation
from twinkle.loss.base import Loss
Expand Down Expand Up @@ -389,14 +390,15 @@ def _attached_head(self, adapter_name):
return heads.get(adapter_name)

def _task_head_state(self, adapter_name: Optional[str]) -> Dict[str, Any]:
"""The adapter's head weights under slot-independent 'task_head.' keys.
"""The head's parameters and calibration buffers under slot-independent keys.

Default (single-model) form reads the head hanging on the host; MultiLora
hosts override this with their slot-pool export."""
head = self._attached_head(adapter_name)
if head is None:
return {}
return {f'task_head.{name}': p.detach().clone().cpu() for name, p in head.named_parameters()}
return {f'task_head.{name}': torch_util.to_local_tensor(tensor).detach().clone().cpu()
for name, tensor in chain(head.named_parameters(), head.named_buffers())}

def _task_head_carries_on_this_rank(self) -> bool:
"""Whether THIS rank is expected to hold the head at all. True by default;
Expand All @@ -409,7 +411,6 @@ def _restore_task_head_state(self, adapter_name: Optional[str], state: Dict[str,
this with their slot-pool import. An empty snapshot passes -- every host checks
the OTHER side's emptiness, never its own (a rank without slots skips, a
checkpoint written head-less stays head-less)."""
import torch
head = self._attached_head(adapter_name)
if head is None:
if state and self._task_head_carries_on_this_rank():
Expand All @@ -418,20 +419,24 @@ def _restore_task_head_state(self, adapter_name: Optional[str], state: Dict[str,
'carries no head to restore them into.')
return
params = dict(head.named_parameters())
expected = {f'task_head.{name}' for name in params}
if set(state) != expected:
tensors = {**params, **dict(head.named_buffers())}
expected = {f'task_head.{name}' for name in tensors}
legacy = {f'task_head.{name}' for name in params}
if set(state) not in (expected, legacy):
raise ValueError(
f'task_head checkpoint keys {sorted(state)} do not match the head it '
f'restores: expected {sorted(expected)}.')
with torch.no_grad():
for name, param in params.items():
value = state[f'task_head.{name}'].to(device=param.device, dtype=param.dtype)
param.data.copy_(value)
for key, value in state.items():
tensor = tensors[key[len('task_head.'):]]
if value.shape != tensor.shape:
raise ValueError(f'task-head shape mismatch for {key}: {value.shape} != {tensor.shape}')
for key, value in state.items():
torch_util.copy_tensor_(tensors[key[len('task_head.'):]], value)

def _save_task_head_checkpoint(self, checkpoint_dir: str, adapter_name: Optional[str]) -> None:
"""Write the adapter's attached head to its own task_head.safetensors.

Keyed 'task_head.<param>' -- slot-independent, so a re-added adapter may land
Keyed 'task_head.<name>' -- slot-independent, so a re-added adapter may land
on a different slot and still restore. Runs on the master rank only; the head
file is tiny, so it writes synchronously alongside the (async) weights."""
state = self._task_head_state(adapter_name)
Expand Down
10 changes: 10 additions & 0 deletions src/twinkle/model/megatron/bridge/mcore.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,16 @@ def build_model_config(self, hf_config: Any, parallel_kwargs: Dict[str, Any], st
# from a compatibility default. mcore's ModelConfig consumes the actual values, not this marker.
kwargs.pop('_strict_model_kwargs', None)
config_kwargs = hf_to_mcore_config(hf_config)
# Preserve a checkpoint's classification head instead of constructing a
# vocabulary-sized language-model output layer and skipping score weights.
if any(name.endswith('ForSequenceClassification')
for name in (getattr(hf_config, 'architectures', None) or [])):
if kwargs.get('task_type', 'seq_cls') != 'seq_cls':
raise ValueError('A sequence-classification checkpoint requires task_type=seq_cls.')
if kwargs.get('num_labels', hf_config.num_labels) != hf_config.num_labels:
raise ValueError('num_labels must match the persisted classification head.')
kwargs.update(task_type='seq_cls', num_labels=hf_config.num_labels,
untie_embeddings_and_output_weights=True)
config_kwargs.update(kwargs)
# per-token-mean grad normalization (mcore default is False; twinkle forces True).
if 'calculate_per_token_loss' not in config_kwargs:
Expand Down
44 changes: 36 additions & 8 deletions src/twinkle/model/megatron/megatron.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
from twinkle.model.micro_batch import (MicroBatchConfig, agree_micro_batch_count,
plan_micro_batches_contiguous, sequence_length)
from twinkle.model.optimizer_group import BaseOptimizerGroup, TrainStatus
from twinkle.model.tasks import get_task_spec_obj
from twinkle.model.tasks import _generative_reranker_score, get_task_spec_obj
from twinkle.patch import Patch, apply_context, apply_patch
from twinkle.perf import timed
from twinkle.processor import InputProcessor
Expand All @@ -59,6 +59,26 @@ def _first_optimizer_lr(optimizer) -> float:
return chained[0].config.lr


def _align_distributed_optimizer_param_order(optimizer) -> None:
"""Match checkpoint indices to MCore's FP32-first main-parameter order.

Some MCore versions index mixed groups in gradient-buffer order, then put
FP32 parameters before low-precision master parameters in the inner optimizer.
A decision head kept in FP32 exposes this mismatch during checkpoint save/load.
"""
from megatron.core.optimizer.distrib_optimizer import DistributedOptimizer

for part in getattr(optimizer, 'chained_optimizers', None) or [optimizer]:
if not isinstance(part, DistributedOptimizer):
continue
fp32_groups = getattr(part, 'model_fp32_groups', ())
low_groups = getattr(part, 'model_float16_groups', ())
for group_index, (fp32, low) in enumerate(zip(fp32_groups, low_groups, strict=True)):
if fp32 and low:
for param_index, param in enumerate([*fp32, *low]):
part.model_param_group_index_map[param] = (group_index, param_index)


def _gather_vocab_parallel(tensor):
"""Differentiable vocab all-gather: ``[n, V/TP]`` -> ``[n, V]`` (identity when TP=1).

Expand Down Expand Up @@ -987,6 +1007,10 @@ def forward_step_func(data_iterator, model):
else:
# Packed: one row [1, total, C]; each segment ends at its last token.
unpacked_logits = output_tensor[0, _last_idx]
if task == 'generative_reranker':
unpacked_logits = _gather_vocab_parallel(unpacked_logits)
unpacked_logits = _generative_reranker_score(
unpacked_logits, self.optimizer_group[adapter_name].template)
elif task == 'value':
# Legacy path -- PPO critic over the bridge-built head: keep its per-token
# V(s_t) riding the logits channel ([b, s]); PPOValueLoss masks to response
Expand Down Expand Up @@ -1528,6 +1552,7 @@ def _create_megatron_optimizer(self, **kwargs):
model_chunks=model_chunks,
config_overrides=config_overrides,
)
_align_distributed_optimizer_param_order(optimizer)
return optimizer

def _create_megatron_scheduler(self, optimizer, lr_decay_steps, max_lr=1e-4, **kwargs):
Expand Down Expand Up @@ -1927,8 +1952,8 @@ def _save_mcore_optimizer(
synchronous path."""
from megatron.core import dist_checkpointing
from megatron.core import parallel_state as mpu
from megatron.core.dist_checkpointing.serialization import TorchDistSaveShardedStrategy
from megatron.core.dist_checkpointing.strategies.fully_parallel import FullyParallelSaveStrategyWrapper
from megatron.core.dist_checkpointing.strategies.torch import TorchDistSaveShardedStrategy

iteration = optimizer_config.cur_step
iter_dir = os.path.join(checkpoint_dir, f'iter_{iteration:07d}')
Expand Down Expand Up @@ -1958,7 +1983,7 @@ def _save_mcore_optimizer(
# megatron-core < 0.19 exposed get_default_save_sharded_strategy(); 0.19 made
# save(sharded_strategy=None) default to TorchDistSaveShardedStrategy itself -- build that
# explicitly, the thread_count tweak and the wrapper below need a concrete instance.
save_strategy = TorchDistSaveShardedStrategy()
save_strategy = TorchDistSaveShardedStrategy(backend='torch_dist', version=1)
if hasattr(save_strategy, 'thread_count'):
save_strategy.thread_count = thread_count
if mpu.get_data_parallel_world_size(with_context_parallel=True) > 1:
Expand All @@ -1970,17 +1995,20 @@ def _save_mcore_optimizer(
# Async staging is only sound under an initialized process group (the metadata finalize
# holds a barrier); a single-process run has no skew to hide anyway.
async_save = kwargs.pop('async_save', False) and dist.is_initialized()
save_parameters = inspect.signature(dist_checkpointing.save).parameters
save_options = {}
if 'async_strategy' in save_parameters:
# Avoid the optional nvidia-resiliency-ext queue on newer MCore.
save_options['async_strategy'] = 'mcore'
if 'content_metadata' in save_parameters:
save_options['content_metadata'] = sharded_sd_metadata
async_request = dist_checkpointing.save(
state_dict,
iter_dir,
save_strategy,
async_sharded_save=async_save,
# mcore's default 'nvrx' strategy needs the nvidia-resiliency-ext package, which is
# not a twinkle dependency; the built-in 'mcore' strategy drives the SAME
# AsyncCallsQueue the consumption path below schedules into.
async_strategy='mcore',
validate_access_integrity=True,
content_metadata=sharded_sd_metadata,
**save_options,
)
if async_request is not None:
# The strategy already staged the tensors to host memory; the queue's worker performs
Expand Down
Loading
Loading