diff --git a/docs/source_en/Usage Guide/Train-as-a-Service.md b/docs/source_en/Usage Guide/Train-as-a-Service.md index c07e072e1..0425583f6 100644 --- a/docs/source_en/Usage Guide/Train-as-a-Service.md +++ b/docs/source_en/Usage Guide/Train-as-a-Service.md @@ -64,6 +64,8 @@ for epoch in range(2): print(f'Saved checkpoint for epoch {epoch} to {result.path}') ``` +On the Transformers backend, finish the current gradient accumulation, optimizer update, and gradient clearing (`zero_grad`) before saving a training checkpoint with Tinker's `save_state()` or Twinkle's `save(..., save_optimizer=True)`. Weights-only exports do not have this restriction. + With the code above, you can train a self-cognition LoRA based on `Qwen/Qwen3.8-27B`. This LoRA will change the model's name and creator to the names specified during training. To perform inference using this LoRA: ```python diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" index fb515944a..94eeb1def 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" @@ -65,6 +65,8 @@ for epoch in range(2): print(f'Saved checkpoint for epoch {epoch} to {result.path}') ``` +使用 Transformers 后端时,请先完成当前梯度累积、优化器更新和梯度清理(`zero_grad`),再调用 Tinker 的 `save_state()` 或 Twinkle 的 `save(..., save_optimizer=True)` 保存训练断点。仅导出模型权重不受此限制。 + 通过上述代码,你可以训练一个原模型为`Qwen/Qwen3.8-27B`的自我认知lora。这个lora会改变模型的名称和制造者为训练时指定的名称。使用这个lora进行推理: ```python diff --git a/src/twinkle/model/base.py b/src/twinkle/model/base.py index 0aaca3fa7..0ef972996 100644 --- a/src/twinkle/model/base.py +++ b/src/twinkle/model/base.py @@ -4,9 +4,10 @@ import shutil from abc import ABC, abstractmethod from concurrent.futures import Future, ThreadPoolExecutor +from itertools import chain from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Protocol, Type, Union -from twinkle import Platform, remote_function +from twinkle import Platform, remote_function, torch_util from twinkle.data_format import InputFeature, ModelOutput from twinkle.hub import HubOperation from twinkle.loss.base import Loss @@ -389,14 +390,15 @@ def _attached_head(self, adapter_name): return heads.get(adapter_name) def _task_head_state(self, adapter_name: Optional[str]) -> Dict[str, Any]: - """The adapter's head weights under slot-independent 'task_head.' keys. + """The head's parameters and calibration buffers under slot-independent keys. Default (single-model) form reads the head hanging on the host; MultiLora hosts override this with their slot-pool export.""" head = self._attached_head(adapter_name) if head is None: return {} - return {f'task_head.{name}': p.detach().clone().cpu() for name, p in head.named_parameters()} + return {f'task_head.{name}': torch_util.to_local_tensor(tensor).detach().clone().cpu() + for name, tensor in chain(head.named_parameters(), head.named_buffers())} def _task_head_carries_on_this_rank(self) -> bool: """Whether THIS rank is expected to hold the head at all. True by default; @@ -409,7 +411,6 @@ def _restore_task_head_state(self, adapter_name: Optional[str], state: Dict[str, this with their slot-pool import. An empty snapshot passes -- every host checks the OTHER side's emptiness, never its own (a rank without slots skips, a checkpoint written head-less stays head-less).""" - import torch head = self._attached_head(adapter_name) if head is None: if state and self._task_head_carries_on_this_rank(): @@ -418,20 +419,24 @@ def _restore_task_head_state(self, adapter_name: Optional[str], state: Dict[str, 'carries no head to restore them into.') return params = dict(head.named_parameters()) - expected = {f'task_head.{name}' for name in params} - if set(state) != expected: + tensors = {**params, **dict(head.named_buffers())} + expected = {f'task_head.{name}' for name in tensors} + legacy = {f'task_head.{name}' for name in params} + if set(state) not in (expected, legacy): raise ValueError( f'task_head checkpoint keys {sorted(state)} do not match the head it ' f'restores: expected {sorted(expected)}.') - with torch.no_grad(): - for name, param in params.items(): - value = state[f'task_head.{name}'].to(device=param.device, dtype=param.dtype) - param.data.copy_(value) + for key, value in state.items(): + tensor = tensors[key[len('task_head.'):]] + if value.shape != tensor.shape: + raise ValueError(f'task-head shape mismatch for {key}: {value.shape} != {tensor.shape}') + for key, value in state.items(): + torch_util.copy_tensor_(tensors[key[len('task_head.'):]], value) def _save_task_head_checkpoint(self, checkpoint_dir: str, adapter_name: Optional[str]) -> None: """Write the adapter's attached head to its own task_head.safetensors. - Keyed 'task_head.' -- slot-independent, so a re-added adapter may land + Keyed 'task_head.' -- slot-independent, so a re-added adapter may land on a different slot and still restore. Runs on the master rank only; the head file is tiny, so it writes synchronously alongside the (async) weights.""" state = self._task_head_state(adapter_name) diff --git a/src/twinkle/model/megatron/bridge/mcore.py b/src/twinkle/model/megatron/bridge/mcore.py index 03771d18b..465f2b38a 100644 --- a/src/twinkle/model/megatron/bridge/mcore.py +++ b/src/twinkle/model/megatron/bridge/mcore.py @@ -25,6 +25,16 @@ def build_model_config(self, hf_config: Any, parallel_kwargs: Dict[str, Any], st # from a compatibility default. mcore's ModelConfig consumes the actual values, not this marker. kwargs.pop('_strict_model_kwargs', None) config_kwargs = hf_to_mcore_config(hf_config) + # Preserve a checkpoint's classification head instead of constructing a + # vocabulary-sized language-model output layer and skipping score weights. + if any(name.endswith('ForSequenceClassification') + for name in (getattr(hf_config, 'architectures', None) or [])): + if kwargs.get('task_type', 'seq_cls') != 'seq_cls': + raise ValueError('A sequence-classification checkpoint requires task_type=seq_cls.') + if kwargs.get('num_labels', hf_config.num_labels) != hf_config.num_labels: + raise ValueError('num_labels must match the persisted classification head.') + kwargs.update(task_type='seq_cls', num_labels=hf_config.num_labels, + untie_embeddings_and_output_weights=True) config_kwargs.update(kwargs) # per-token-mean grad normalization (mcore default is False; twinkle forces True). if 'calculate_per_token_loss' not in config_kwargs: diff --git a/src/twinkle/model/megatron/megatron.py b/src/twinkle/model/megatron/megatron.py index ef89ab462..e18889f41 100644 --- a/src/twinkle/model/megatron/megatron.py +++ b/src/twinkle/model/megatron/megatron.py @@ -36,7 +36,7 @@ from twinkle.model.micro_batch import (MicroBatchConfig, agree_micro_batch_count, plan_micro_batches_contiguous, sequence_length) from twinkle.model.optimizer_group import BaseOptimizerGroup, TrainStatus -from twinkle.model.tasks import get_task_spec_obj +from twinkle.model.tasks import _generative_reranker_score, get_task_spec_obj from twinkle.patch import Patch, apply_context, apply_patch from twinkle.perf import timed from twinkle.processor import InputProcessor @@ -59,6 +59,26 @@ def _first_optimizer_lr(optimizer) -> float: return chained[0].config.lr +def _align_distributed_optimizer_param_order(optimizer) -> None: + """Match checkpoint indices to MCore's FP32-first main-parameter order. + + Some MCore versions index mixed groups in gradient-buffer order, then put + FP32 parameters before low-precision master parameters in the inner optimizer. + A decision head kept in FP32 exposes this mismatch during checkpoint save/load. + """ + from megatron.core.optimizer.distrib_optimizer import DistributedOptimizer + + for part in getattr(optimizer, 'chained_optimizers', None) or [optimizer]: + if not isinstance(part, DistributedOptimizer): + continue + fp32_groups = getattr(part, 'model_fp32_groups', ()) + low_groups = getattr(part, 'model_float16_groups', ()) + for group_index, (fp32, low) in enumerate(zip(fp32_groups, low_groups, strict=True)): + if fp32 and low: + for param_index, param in enumerate([*fp32, *low]): + part.model_param_group_index_map[param] = (group_index, param_index) + + def _gather_vocab_parallel(tensor): """Differentiable vocab all-gather: ``[n, V/TP]`` -> ``[n, V]`` (identity when TP=1). @@ -987,6 +1007,10 @@ def forward_step_func(data_iterator, model): else: # Packed: one row [1, total, C]; each segment ends at its last token. unpacked_logits = output_tensor[0, _last_idx] + if task == 'generative_reranker': + unpacked_logits = _gather_vocab_parallel(unpacked_logits) + unpacked_logits = _generative_reranker_score( + unpacked_logits, self.optimizer_group[adapter_name].template) elif task == 'value': # Legacy path -- PPO critic over the bridge-built head: keep its per-token # V(s_t) riding the logits channel ([b, s]); PPOValueLoss masks to response @@ -1528,6 +1552,7 @@ def _create_megatron_optimizer(self, **kwargs): model_chunks=model_chunks, config_overrides=config_overrides, ) + _align_distributed_optimizer_param_order(optimizer) return optimizer def _create_megatron_scheduler(self, optimizer, lr_decay_steps, max_lr=1e-4, **kwargs): @@ -1927,8 +1952,8 @@ def _save_mcore_optimizer( synchronous path.""" from megatron.core import dist_checkpointing from megatron.core import parallel_state as mpu - from megatron.core.dist_checkpointing.serialization import TorchDistSaveShardedStrategy from megatron.core.dist_checkpointing.strategies.fully_parallel import FullyParallelSaveStrategyWrapper + from megatron.core.dist_checkpointing.strategies.torch import TorchDistSaveShardedStrategy iteration = optimizer_config.cur_step iter_dir = os.path.join(checkpoint_dir, f'iter_{iteration:07d}') @@ -1958,7 +1983,7 @@ def _save_mcore_optimizer( # megatron-core < 0.19 exposed get_default_save_sharded_strategy(); 0.19 made # save(sharded_strategy=None) default to TorchDistSaveShardedStrategy itself -- build that # explicitly, the thread_count tweak and the wrapper below need a concrete instance. - save_strategy = TorchDistSaveShardedStrategy() + save_strategy = TorchDistSaveShardedStrategy(backend='torch_dist', version=1) if hasattr(save_strategy, 'thread_count'): save_strategy.thread_count = thread_count if mpu.get_data_parallel_world_size(with_context_parallel=True) > 1: @@ -1970,17 +1995,20 @@ def _save_mcore_optimizer( # Async staging is only sound under an initialized process group (the metadata finalize # holds a barrier); a single-process run has no skew to hide anyway. async_save = kwargs.pop('async_save', False) and dist.is_initialized() + save_parameters = inspect.signature(dist_checkpointing.save).parameters + save_options = {} + if 'async_strategy' in save_parameters: + # Avoid the optional nvidia-resiliency-ext queue on newer MCore. + save_options['async_strategy'] = 'mcore' + if 'content_metadata' in save_parameters: + save_options['content_metadata'] = sharded_sd_metadata async_request = dist_checkpointing.save( state_dict, iter_dir, save_strategy, async_sharded_save=async_save, - # mcore's default 'nvrx' strategy needs the nvidia-resiliency-ext package, which is - # not a twinkle dependency; the built-in 'mcore' strategy drives the SAME - # AsyncCallsQueue the consumption path below schedules into. - async_strategy='mcore', validate_access_integrity=True, - content_metadata=sharded_sd_metadata, + **save_options, ) if async_request is not None: # The strategy already staged the tensors to host memory; the queue's worker performs diff --git a/src/twinkle/model/multi_lora.py b/src/twinkle/model/multi_lora.py index 1351a35a8..dd08902f5 100644 --- a/src/twinkle/model/multi_lora.py +++ b/src/twinkle/model/multi_lora.py @@ -3,9 +3,9 @@ from contextlib import contextmanager, nullcontext from copy import deepcopy from dataclasses import dataclass, field +from itertools import chain from peft import LoraConfig, PeftModel, get_peft_model from peft.tuners.lora import Embedding, Linear, LoraLayer -from torch.distributed.tensor import distribute_tensor from types import MethodType from typing import Any, Callable, Dict, List, Optional, Union @@ -56,7 +56,8 @@ def build(self, host: torch.nn.Module, build_head: Callable[[], torch.nn.Module] head = build_head() setattr(host, f'task_head_slot_{slot}', head) self._initial_weights[slot] = { - name: parameter.detach().clone().cpu() for name, parameter in head.named_parameters()} + name: tensor.detach().clone().cpu() + for name, tensor in chain(head.named_parameters(), head.named_buffers())} host.task_heads = {} def bind(self, adapter_name: str, host: torch.nn.Module) -> None: @@ -93,7 +94,9 @@ def release(self, adapter_name: str, host: torch.nn.Module, if head is None: return with torch.no_grad(): - for name, parameter in head.named_parameters(): + # Calibration buffers can be nonpersistent in the family head. They still + # belong to this tenant and must not leak when the physical slot is reused. + for name, parameter in chain(head.named_parameters(), head.named_buffers()): saved = self._initial_weights.get(slot, {}).get(name) if saved is not None: write_param(parameter, saved) @@ -118,7 +121,7 @@ def export_state(self, adapter_name: str, host: torch.nn.Module, return {} return { f'task_head.{name}': read_param(parameter).detach().clone().cpu() - for name, parameter in head.named_parameters() + for name, parameter in chain(head.named_parameters(), head.named_buffers()) } def import_state(self, adapter_name: str, host: torch.nn.Module, state: Dict[str, torch.Tensor], @@ -142,16 +145,25 @@ def import_state(self, adapter_name: str, host: torch.nn.Module, state: Dict[str # lives on the last stage; every rank binds the same slot index. return parameters = dict(head.named_parameters()) + tensors = {**parameters, **dict(head.named_buffers())} covered = {key[len(prefix):] for key in head_keys} - if covered != set(parameters): + # Parameter-only checkpoints predate calibration snapshots; keep them loadable. + if covered not in (set(parameters), set(tensors)): raise ValueError( f'checkpoint task-head keys {sorted(covered)} do not match the bound head\'s ' - f'parameters {sorted(parameters)}: the head architecture changed since the ' + f'tensors {sorted(tensors)}: the head architecture changed since the ' 'checkpoint was written.') + for key in head_keys: + tensor = tensors[key[len(prefix):]] + shapes = {tensor.shape} + if hasattr(tensor, 'to_local'): + shapes.add(tensor.to_local().shape) + if state[key].shape not in shapes: + raise ValueError(f'task-head shape mismatch for {key}: {state[key].shape} != {tensor.shape}') with torch.no_grad(): for key in head_keys: name = key[len(prefix):] - parameter = parameters[name] + parameter = tensors[name] write_param(parameter, state[key]) @@ -176,39 +188,13 @@ def _get_available_lora(self) -> Optional[LoraTenant]: def _read_param_tensor(self, parameter): return torch_util.to_local_tensor(parameter) - @staticmethod - def _is_distributed_param(parameter): - return hasattr(parameter, 'device_mesh') and hasattr(parameter, 'placements') - @staticmethod def _is_target_parameter_lora_name(name: str) -> bool: return '._twinkle_lora_' in name def _write_param_tensor(self, parameter, value): - if value is None: - return - value = value.detach().to(dtype=parameter.dtype) - if self._is_distributed_param(parameter): - if self._is_distributed_param(value): - parameter.data.copy_(value.to(parameter.device)) - return - - local_param = parameter.to_local() if hasattr(parameter, 'to_local') else None - parameter_shape = tuple(parameter.shape) - value_shape = tuple(value.shape) - if local_param is not None: - local_shape = tuple(local_param.shape) - if value_shape == local_shape and value_shape != parameter_shape: - local_param.copy_(value.to(local_param.device)) - return - if value_shape != parameter_shape: - raise ValueError( - f'Cannot write tensor with shape {value_shape} to distributed parameter with global shape ' - f'{parameter_shape} and local shape {local_shape}') - value = distribute_tensor(value.to(parameter.device), parameter.device_mesh, parameter.placements) - else: - value = value.to(parameter.device) - parameter.data.copy_(value) + if value is not None: + torch_util.copy_tensor_(parameter, value) @staticmethod def _slice_rank_tensor(name: str, tensor, rank: int): @@ -492,9 +478,9 @@ def match_target_modules( # Strip LoRA-specific suffixes (e.g. ".lora_A.default.weight") so that # a full parameter name like "model.layers.0.attn.proj.lora_A.default.weight" # can be matched against target_modules like ["attn.proj"]. - cleaned = module_name - if '.lora_' in module_name: - cleaned = re.sub(r'\.lora_\w+(\.[\w-]+)*$', '', module_name) + cleaned = module_name.removeprefix('base_model.model.') + if '.lora_' in cleaned: + cleaned = re.sub(r'\.lora_\w+(\.[\w-]+)*$', '', cleaned) if isinstance(target_modules, str): return re.fullmatch(target_modules, cleaned) is not None @@ -676,6 +662,14 @@ def patch(self, low_cpu_mem_usage = module_device.type == 'meta' base_config = kwargs.get('lora_config', None) + megatron_targets = None + if isinstance(module, list): + requested = base_config.target_modules if base_config is not None else target_modules + if requested: + from .megatron import MegatronModel + requested = [requested] if isinstance(requested, str) else list(requested) + # Resolve each pristine chunk once; adding a slot changes its module tree. + megatron_targets = [MegatronModel.get_target_modules(chunk, requested) for chunk in module] for i in range(self.max_loras): config = deepcopy(base_config) if base_config is not None else None if config is None: @@ -703,22 +697,14 @@ def _patch_peft(_module): self._patch_lora_forward(name, submodule) return _module - def _patch_megatron(_module): - # Expand target_modules (e.g., 'all-linear' -> actual module names) + def _patch_megatron(_module, targets): _config = deepcopy(config) + if targets is not None: + _config.target_modules = targets if isinstance(_module, PeftModel): _module.add_adapter(lora_tenant.adapter_name, _config, low_cpu_mem_usage=low_cpu_mem_usage) self._autocast_adapter_dtype(_module, lora_tenant.adapter_name) else: - # TODO first wrap needs parse target_modules, need to fix later - if _config.target_modules: - if isinstance(_config.target_modules, str): - target_modules = [_config.target_modules] - else: - target_modules = list(_config.target_modules) - - from .megatron import MegatronModel - _config.target_modules = MegatronModel.get_target_modules(_module, target_modules) _module = get_peft_model( _module, _config, lora_tenant.adapter_name, low_cpu_mem_usage=low_cpu_mem_usage) @@ -728,7 +714,8 @@ def _patch_megatron(_module): return _module if isinstance(module, list): - module = [_patch_megatron(_m) for _m in module] + module = [_patch_megatron(chunk, megatron_targets[index] if megatron_targets is not None else None) + for index, chunk in enumerate(module)] else: module = _patch_peft(module) diff --git a/src/twinkle/model/tasks.py b/src/twinkle/model/tasks.py index 4a9fa1408..f60d58e65 100644 --- a/src/twinkle/model/tasks.py +++ b/src/twinkle/model/tasks.py @@ -320,6 +320,12 @@ class Decision(TaskSpec): megatron_context = _no_patch +def _generative_reranker_score(logits, template): + """Return the positive-minus-negative token logit for a generative reranker.""" + positive, negative = _resolve_generative_reranker_token_ids(template) + return logits[..., positive:positive + 1] - logits[..., negative:negative + 1] + + def _resolve_generative_reranker_token_ids(template): """Resolve the positive/negative token ids for a generative reranker from the template tokenizer. diff --git a/src/twinkle/model/transformers/multi_lora_transformers.py b/src/twinkle/model/transformers/multi_lora_transformers.py index ae71f0d8b..599575a11 100644 --- a/src/twinkle/model/transformers/multi_lora_transformers.py +++ b/src/twinkle/model/transformers/multi_lora_transformers.py @@ -104,12 +104,13 @@ def __init__( else: from transformers import AutoConfig self.hf_config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) - if model_cls is None and hasattr(self.hf_config, 'architectures'): - model_cls = self.hf_config.architectures[0] - if model_cls is None: - model_cls = AutoModelForCausalLM - if isinstance(model_cls, str): - model_cls = getattr(transformers, model_cls) + if model_loader is None or model_id is None: + if model_cls is None and getattr(self.hf_config, 'architectures', None): + model_cls = self.hf_config.architectures[0] + if model_cls is None: + model_cls = AutoModelForCausalLM + if isinstance(model_cls, str): + model_cls = getattr(transformers, model_cls) if model_loader is not None and model_id is not None: processor = model_loader.process_tokenizer(model_loader.build_processor(model_id, self.hf_config)) self._default_tokenizer = processor @@ -358,6 +359,12 @@ def add_adapter_to_model(self, adapter_name: str, config_or_dir: Union[PeftConfi config = self._copy_lora_config(config_or_dir) if config.modules_to_save: raise ValueError('modules_to_save is not supported for a multi-tenant LoRA adapter.') + self._expand_multimodal_lora_target( + config, + self.strategy.unwrap_model(self.model), + freeze_llm=kwargs.pop('freeze_llm', False), + freeze_vit=kwargs.pop('freeze_vit', True), + freeze_aligner=kwargs.pop('freeze_aligner', True)) self.multi_adapter.validate_tenant_target_modules(config.target_modules, getattr(config, 'target_parameters', None)) self._register_adapter(adapter_name, config, **kwargs) diff --git a/src/twinkle/model/transformers/strategy/native_fsdp.py b/src/twinkle/model/transformers/strategy/native_fsdp.py index 672265fc6..3229fb1ef 100644 --- a/src/twinkle/model/transformers/strategy/native_fsdp.py +++ b/src/twinkle/model/transformers/strategy/native_fsdp.py @@ -24,6 +24,15 @@ PEFT_BASE_LAYER_SEGMENT = 'base_layer' +def _shard_fp32_modules(model, *, mesh, reshard_after_forward): + # FSDP parameter groups require one storage dtype; keep FP32 modules separate from the BF16 group. + for module in model.modules(): + if (not getattr(module, 'keep_fp32', False) or isinstance(module, FSDPModule) + or next(module.parameters(), None) is None): + continue + fully_shard(module, mesh=mesh, reshard_after_forward=reshard_after_forward, mp_policy=_build_mp_policy('no')) + + class NativeFSDPStrategy(TransformerStrategyBase): def __init__(self, @@ -193,6 +202,8 @@ def wrap_model(self, model, optimizer=None, *additional_optimizers): world_size = self.device_mesh.world_size ep_fsdp_mesh_1d = self.ep_fsdp_device_mesh['ep_fsdp'] if ep_enabled else None + _shard_fp32_modules(model, mesh=fsdp_mesh, reshard_after_forward=reshard_after_forward) + for layer_mod, experts_mod in layer_pairs: layer_mod._fsdp_modules = [] @@ -293,6 +304,10 @@ def save_optimizer_checkpoint(self, model, optimizer, output_path: str, *, param options=self._prepare_optimizer_state_dict_options(for_load=False), ) if Platform.is_master(): + # DCP requires an entry for every optimizer parameter; Adam leaves unused parameters uninitialized. + for group in optim_state['param_groups']: + for name in group['params']: + optim_state['state'].setdefault(name, {}) remap_optimizer_state_names(optim_state, param_name_mapping or {}) torch.save(optim_state, output_path) @@ -311,15 +326,25 @@ def load_optimizer_checkpoint(self, model, optimizer, input_path: str, *, param_ from torch.distributed.checkpoint.state_dict import set_optimizer_state_dict optim_state = {} + empty_states = None if Platform.is_master(): optim_state = torch.load(input_path, map_location='cpu', weights_only=True) remap_optimizer_state_names(optim_state, param_name_mapping or {}) + empty_states = {name for name, value in optim_state['state'].items() if not value} + if dist.is_initialized(): + payload = [empty_states] + dist.broadcast_object_list(payload, src=0) + empty_states = payload[0] set_optimizer_state_dict( model, optimizer, optim_state, options=self._prepare_optimizer_state_dict_options(for_load=True), ) + # DCP initializes every Adam slot before loading; explicitly empty snapshots remain lazy. + parameters = dict(model.named_parameters()) + for name in empty_states: + optimizer.state.pop(parameters[name], None) def get_ep_clip_kwargs(self, model) -> Dict[str, Any]: """Return EP-aware kwargs for normalize_and_clip_grad_norm.""" diff --git a/src/twinkle/model/transformers/transformers.py b/src/twinkle/model/transformers/transformers.py index 0a8fc1d94..cdb0717f0 100644 --- a/src/twinkle/model/transformers/transformers.py +++ b/src/twinkle/model/transformers/transformers.py @@ -337,12 +337,13 @@ def __init__( logger.warning_once('loading %s with trust_remote_code=True: the checkpoint\'s own Python code ' 'will execute inside this process. Only use checkpoints you trust.', model_id) self.hf_config = AutoConfig.from_pretrained(model_id, trust_remote_code=True) - if model_cls is None and hasattr(self.hf_config, 'architectures'): - model_cls = self.hf_config.architectures[0] - if model_cls is None: - model_cls = AutoModelForCausalLM - if isinstance(model_cls, str): - model_cls = getattr(transformers, model_cls) + if model_loader is None or model_id is None: + if model_cls is None and getattr(self.hf_config, 'architectures', None): + model_cls = self.hf_config.architectures[0] + if model_cls is None: + model_cls = AutoModelForCausalLM + if isinstance(model_cls, str): + model_cls = getattr(transformers, model_cls) if model_loader is not None and model_id is not None: # Full takeover: the loader builds the processor, then the pretrained model (its own # model_cls, trust_remote_code and dynamic-module handling), then post-processes it @@ -2468,7 +2469,8 @@ def save(self, name: Optional[str] = None, output_dir: Optional[str] = None, int interval: Save each interval steps. **kwargs: adapter_name: Lora adapter name. - save_optimizer: Whether to save optimizer state. + save_optimizer: Whether to save optimizer state. Requires a complete gradient + accumulation window; pending gradients are not included in checkpoints. safe_serialization: Save weights with safetensors instead of torch serialization. max_shard_size: Maximum size of each full-model weight shard. save_total_limit: Maximum number of completed checkpoint directories to retain. @@ -2488,6 +2490,8 @@ def save(self, name: Optional[str] = None, output_dir: Optional[str] = None, int checkpoint_dir = os.path.join(output_dir, name) if optimizer_config.cur_step % interval != 0: return + if kwargs.get('save_optimizer', False): + self._validate_training_state_save(adapter_name) if isinstance(self.strategy, NativeFSDPStrategy) and self.strategy.use_rank0_pretrained_broadcast(): # Initial adapter snapshots precede the first forward. Materialize on every # rank before state-dict collectives: non-source ranks still hold meta tensors. @@ -2591,9 +2595,22 @@ def _save_optimizer(self, output_dir, **kwargs): if lr_scheduler is not None: torch.save(lr_scheduler.state_dict(), os.path.join(output_dir, 'scheduler.pt')) + def _validate_training_state_save(self, adapter_name): + optimizer_config = self.optimizer_group[adapter_name] + optimizer_config.require_finalized('save training state') + optimizer = optimizer_config.optimizer + pending_gradients = optimizer is not None and any( + parameter.grad is not None for group in optimizer.param_groups for parameter in group['params']) + if optimizer_config.cur_step % optimizer_config.gradient_accumulation_steps or pending_gradients: + raise ValueError( + 'save_optimizer=True requires a complete gradient accumulation window; ' + 'finish the pending backward/optimizer steps and zero_grad before saving, or save weights only.') + def _save_training_state(self, output_dir, **kwargs): adapter_name = kwargs.pop('adapter_name', _default_adapter_name) defer_trainer_state = kwargs.pop('defer_trainer_state', False) + self._validate_training_state_save(adapter_name) + optimizer_config = self.optimizer_group[adapter_name] # The optimizer/scheduler files are written before _write_weights() gets a chance to # create the checkpoint dir via save_pretrained; torch.save then raises # "Parent directory ... does not exist". This lives HERE (not at the top of save()) so it @@ -2601,8 +2618,6 @@ def _save_training_state(self, output_dir, **kwargs): os.makedirs(output_dir, exist_ok=True) self._save_optimizer(output_dir, adapter_name=adapter_name) - optimizer_config = self.optimizer_group[adapter_name] - # RNG state is per rank, so it is written here rather than below the master-only return: # each rank's generator has advanced by its own dropout draws, and one master-written file # would replay rank 0's stream on every rank. Wherever the forward is stochastic @@ -2838,6 +2853,10 @@ def _restore_training_state(self, checkpoint_dir, *, adapter_name=''): trainer_state_path = os.path.join(checkpoint_dir, 'trainer_state.json') with open(trainer_state_path) as f: trainer_state = json.load(f) + if trainer_state['cur_step'] % trainer_state['gradient_accumulation_steps']: + raise ValueError( + 'Checkpoint was saved during gradient accumulation without pending gradients; ' + 'use resume_only_model=True to start fresh, or resume a complete accumulation window.') adapter_name = adapter_name or self._get_default_group() optimizer_config = self.optimizer_group[adapter_name] diff --git a/src/twinkle/server/config/training.py b/src/twinkle/server/config/training.py index 259053050..fa7262a5c 100644 --- a/src/twinkle/server/config/training.py +++ b/src/twinkle/server/config/training.py @@ -60,7 +60,7 @@ class TrainingArguments(BaseModel): mixed_precision: Literal["no", "fp8", "fp16", "bf16"] | None = None # The readout family names the head task this deployment serves (see # ``twinkle/model/tasks.py``); ``causal_lm`` carries no head. - task_type: Literal["causal_lm", "seq_cls", "reranker", "value"] | None = None + task_type: Literal["causal_lm", "embedding", "seq_cls", "reranker", "value", "decision"] | None = None num_labels: int | None = Field(default=None, gt=0, strict=True) language_model_only: bool | None = Field(default=None, strict=True) @@ -70,6 +70,8 @@ def _validate_head(self): raise ValueError('task_type="seq_cls" requires num_labels') if self.num_labels is not None and self.task_type not in ("seq_cls", "reranker", "value"): raise ValueError('num_labels requires a readout task_type (seq_cls / reranker / value)') + if self.task_type == "decision" and self.model_loader is None: + raise ValueError('task_type="decision" requires a family model_loader') return self @model_validator(mode="after") @@ -78,6 +80,24 @@ def _validate_replicas(self): raise ValueError("min_replicas must not exceed max_replicas") return self + def validate_backend(self, backend: str, *, mode: str, hybrid=None) -> None: + if self.strategy is not None and backend != "transformers": + raise ValueError("strategy requires the transformers backend") + if self.gradient_checkpointing is not None and backend != "transformers": + raise ValueError("gradient_checkpointing requires the transformers backend") + if self.mixed_precision is not None and backend == "mock": + raise ValueError("mixed_precision requires the transformers or megatron backend") + if hybrid is not None: + if backend != "transformers": + raise ValueError("hybrid requires the transformers backend") + if mode == "full": + raise ValueError('hybrid is not supported with train_mode="full"') + if self.model_cls is not None and (backend != "transformers" or mode not in ("lora", "full") + or hybrid is not None): + raise ValueError("model_cls requires a transformers LoRA/full deployment") + if self.model_loader is not None and backend == "mock": + raise ValueError("model_loader requires the transformers or megatron backend") + class LoraArguments(BaseModel): """LoRA parameters shared by YAML and LoRA instance configuration.""" @@ -107,22 +127,7 @@ class ModelArgs(TrainingArguments, LoraArguments): @model_validator(mode="after") def _validate_spectral_backend(self): - if self.strategy is not None and self.backend != "transformers": - raise ValueError("strategy is only supported by the transformers backend") - if self.backend != "transformers" and ( - self.gradient_checkpointing is not None or self.mixed_precision is not None - ): - raise ValueError("gradient_checkpointing and mixed_precision require the transformers backend") - if self.hybrid is not None: - if self.backend != "transformers": - raise ValueError("hybrid is only supported by the transformers backend") - if self.train_mode == "full": - raise ValueError('hybrid is not supported with train_mode="full"') - if self.model_cls is not None: - if self.backend != "transformers" or self.train_mode != "lora" or self.hybrid is not None: - raise ValueError("model_cls requires a transformers LoRA deployment without hybrid") - if self.model_loader is not None and self.backend != "transformers": - raise ValueError("model_loader requires the transformers backend") + self.validate_backend(self.backend, mode=self.train_mode, hybrid=self.hybrid) if self.language_model_only is not None and self.backend != "megatron": raise ValueError("language_model_only requires the megatron backend") return self @@ -206,19 +211,7 @@ def _validate_backend(self): raise ValueError("megatron training instances require at least 2 devices") if self.config.device_mesh.device_type not in (None, self.device_type): raise ValueError("device_mesh.device_type must match device_type") - if self.config.strategy is not None and self.backend != "transformers": - raise ValueError("strategy requires the transformers backend") - if self.backend != "transformers" and ( - self.config.gradient_checkpointing is not None or self.config.mixed_precision is not None - ): - raise ValueError("gradient_checkpointing and mixed_precision require the transformers backend") - if isinstance(self.config, LoraTrainingConfig) and self.config.hybrid and self.backend != "transformers": - raise ValueError("hybrid requires the transformers backend") - if self.config.model_cls is not None: - if self.backend != "transformers" or self.mode != "lora" or self.config.hybrid is not None: - raise ValueError("model_cls requires a transformers LoRA deployment without hybrid") - if self.config.model_loader is not None and self.backend != "transformers": - raise ValueError("model_loader requires the transformers backend") + self.config.validate_backend(self.backend, mode=self.mode, hybrid=getattr(self.config, "hybrid", None)) return self @property diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py index 318886216..db0a44bb8 100644 --- a/src/twinkle/server/lifecycle/submit.py +++ b/src/twinkle/server/lifecycle/submit.py @@ -15,7 +15,7 @@ from twinkle.data_format import InputFeature, Trajectory, is_encoded from twinkle.protocol.json_utils import json_safe from twinkle.protocol.types.base import FieldRole, fields_with_role -from twinkle.protocol.types.data import export_batch +from twinkle.protocol.types.data import VLM_TENSOR_FIELDS, export_batch from twinkle.protocol.types.lifecycle import TaskEnvelope from twinkle.protocol.types.model import (AdapterRequest, DataPlaneForwardRequest, ForwardBackwardTaskRequest, TrainingRequest) @@ -46,8 +46,8 @@ def to_backend_inputs(inputs: Any, *, single: bool = False) -> Any: With ``single=True`` exactly one object is returned (the streaming path accepts only one input) and a batch of any other size is a ``ValueError``. Plain dicts pass - through unchanged: the data-plane path resolves rows itself and never goes through - the wire schema. + through with their encoded media tensors restored: JSON lists cannot be passed to + a template's tensor collator, nor concatenated row-wise without losing shape. """ entries = export_batch(inputs) if isinstance(inputs, list) else inputs if single: @@ -67,7 +67,16 @@ def to_backend_inputs(inputs: Any, *, single: bool = False) -> Any: def _as_backend_entry(entry: dict[str, Any]) -> Any: """One exported entry as its ``TypedDict`` shape.""" - return InputFeature(**entry) if is_encoded(entry) else Trajectory(**entry) + if not is_encoded(entry): + return Trajectory(**entry) + result = InputFeature(**entry) + media = VLM_TENSOR_FIELDS.intersection(result) + if media: + import torch + for key in media: + if result[key] is not None and not isinstance(result[key], torch.Tensor): + result[key] = torch.as_tensor(result[key]) + return result def backend_kwargs(body: Any) -> dict[str, Any]: diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index e40acb753..6dddf291f 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -53,10 +53,23 @@ def _validate_model_task(kw: dict[str, Any]) -> None: task_type = kw.pop('task_type', None) if task_type is None: return + if task_type == 'decision': + loader = kw.get('model_loader') + head_factory = getattr(loader, 'task_head_loader', None) + if getattr(loader, 'task_type', None) != 'decision' or not callable(head_factory): + raise ValueError('decision requires a family model_loader with task_head_loader()') + kw['task'] = 'decision' + kw['task_head_loader'] = head_factory() + return + from pathlib import Path from transformers import AutoConfig from twinkle.hub import HubOperation path = HubOperation.download_model(kw['model_id']) + if task_type == 'embedding' and kw.get('model_cls') is None and (Path(path) / 'modules.json').is_file(): + raise ValueError( + 'native sentence-transformers checkpoint requires model_cls; ' + 'the generic embedding wrapper cannot preserve its module pipeline') loader = kw.get('model_loader') config = (loader.process_config(loader.build_config(path)) if loader is not None else AutoConfig.from_pretrained(path, trust_remote_code=trust_remote_code())) @@ -67,7 +80,6 @@ def _validate_model_task(kw: dict[str, Any]) -> None: if classification: if num_labels is None or getattr(config, 'num_labels', None) != num_labels: raise ValueError('num_labels does not match the persisted classification head') - from pathlib import Path from safetensors import safe_open head_found = False for file in Path(path).glob('model*.safetensors'): @@ -112,19 +124,30 @@ def _make_model_class(base: type) -> type: def _make_transformers_model(kw: dict[str, Any]) -> Any: train_mode = kw.pop('train_mode', 'lora') + # Family-owned module paths keep all-linear LoRA on the language-model side + # of a multimodal backbone, as on the loader's native training path. + arch = getattr(kw.get('model_loader'), 'model_arch', None) + if arch is not None: + kw['model_arch'] = {name: getattr(arch, name) for name in + ('language_model', 'vision_tower', 'aligner', 'lm_head') + if getattr(arch, name, None)} # ``model_cls`` arrives as plain data and is resolved here, so the class object never reaches the # constructor -- it would either be rejected by the remote-class layer in safe mode - # (``TWINKLE_TRUST_REMOTE_CODE=0``) or leak into every worker's constructor kwargs. The schema only - # allows it for a LoRA-mode transformers deployment. + # (``TWINKLE_TRUST_REMOTE_CODE=0``) or leak into every worker's constructor kwargs. The schema + # allows it for non-hybrid transformers deployments. model_cls_spec = kw.pop('model_cls', None) if train_mode == 'full': from .backends.transformers_model import TwinkleCompatFullTransformersModel for key in _LORA_ONLY_CTOR_KWARGS: kw.pop(key, None) - # The plain TransformersModel attaches heads at add_adapter time -- a full-mode - # deployment has no adapter, so an attached-head task_type has nothing to ride on. - if kw.pop('task', None) is not None: - raise ValueError('the transformers full-mode backend serves no task head; use train_mode: lora') + if kw.get('task') == 'decision' and getattr(kw.get('model_loader'), 'task_type', None) == 'decision': + # The family loader attaches its single head while constructing the full model. + kw.pop('task') + kw.pop('task_head_loader', None) + if kw.get('task') is None: + kw.pop('num_labels', None) + if model_cls_spec is not None: + return _make_model_class(_resolve_model_cls(model_cls_spec))(**kw) return TwinkleCompatFullTransformersModel(**kw) from .backends.transformers_model import (TwinkleCompatSpectralHybridTransformersModel, TwinkleCompatTransformersModel) diff --git a/src/twinkle/server/model/backends/transformers_model.py b/src/twinkle/server/model/backends/transformers_model.py index 2c175e692..7b3aa5b7a 100644 --- a/src/twinkle/server/model/backends/transformers_model.py +++ b/src/twinkle/server/model/backends/transformers_model.py @@ -14,7 +14,7 @@ """ from tinker import types -from twinkle import remote_class, remote_function +from twinkle import DeviceMesh, remote_class, remote_function from twinkle.data_format import InputFeature, Trajectory from twinkle.model import MultiLoraTransformersModel, SpectralHybridTransformersModel from twinkle.model.transformers import TransformersModel @@ -133,3 +133,9 @@ class TwinkleCompatFullTransformersModel(_TransformersTinkerCompatMixin, Transfo target the empty-string default optimizer group created by ``TransformersModel.__init__``. """ + + def __init__(self, *args, device_mesh: DeviceMesh = None, task=None, num_labels=None, + task_head_loader=None, **kwargs): + super().__init__(*args, device_mesh=device_mesh, **kwargs) + if task is not None or task_head_loader is not None: + self._add_task_head('', task or 'decision', task_head_loader, num_labels) diff --git a/src/twinkle/signals/response.py b/src/twinkle/signals/response.py index b44fccad1..9eb322a6c 100644 --- a/src/twinkle/signals/response.py +++ b/src/twinkle/signals/response.py @@ -45,18 +45,22 @@ def response_token_scores(outputs: Any, features: List[Dict[str, Any]], *, outpu ``outputs`` is the forward's return (a pending handle is resolved here) -- a dict whose ``output_key`` channel (``'logps'`` for a causal scorer, ``'logits'`` / ``'values'`` for a value - head) is a ``[B, T]`` tensor or a bare tensor. Full-sequence, right-padded rows are indexed at - each feature's own response positions, so right-padding never shifts them. + head) may contain a batch tensor or variable-length microbatches, including their JSON form. + Full-sequence, right-padded rows are indexed at each feature's response positions. """ outputs = outputs.result() if hasattr(outputs, 'result') else outputs values = outputs.get(output_key) if isinstance(outputs, dict) else outputs if values is None: raise RuntimeError(f'the forward returned no {output_key!r} channel.') - values = torch.as_tensor(values).detach().float().cpu() - if values.dim() == 1: - values = values.unsqueeze(0) - if values.shape[0] != len(features): - raise RuntimeError(f'forward returned {values.shape[0]} rows for {len(features)} features; ' + if not (isinstance(values, (list, tuple)) and values + and isinstance(values[0], (list, tuple, np.ndarray, torch.Tensor))): + values = [values] + rows = [] + for batch in values: + batch = torch.as_tensor(batch).detach().float().cpu() + rows.extend(batch.unbind(0) if batch.dim() > 1 else [batch]) + if len(rows) != len(features): + raise RuntimeError(f'forward returned {len(rows)} rows for {len(features)} features; ' 'per-row alignment is impossible.') - return [_row_scores(values[i].reshape(-1), response_positions(feature), feature) - for i, feature in enumerate(features)] + return [_row_scores(row.reshape(-1), response_positions(feature), feature) + for row, feature in zip(rows, features, strict=True)] diff --git a/src/twinkle/utils/framework.py b/src/twinkle/utils/framework.py index f72c523e3..11c32e478 100644 --- a/src/twinkle/utils/framework.py +++ b/src/twinkle/utils/framework.py @@ -178,6 +178,35 @@ def to_local_tensor(tensor: 'torch.Tensor') -> 'torch.Tensor': return tensor.to_local() return tensor + @staticmethod + def copy_tensor_(tensor: 'torch.Tensor', value: 'torch.Tensor') -> None: + """Copy a full tensor or a saved local shard in place, preserving the target layout.""" + import torch + + with torch.no_grad(): + value = value.detach().to(dtype=tensor.dtype) + if hasattr(tensor, 'device_mesh') and hasattr(tensor, 'placements'): + if hasattr(value, 'device_mesh') and hasattr(value, 'placements'): + tensor.copy_(value.to(tensor.device)) + return + local = tensor.to_local() if hasattr(tensor, 'to_local') else None + global_shape, value_shape = tuple(tensor.shape), tuple(value.shape) + if local is not None: + local_shape = tuple(local.shape) + if value_shape == local_shape and value_shape != global_shape: + local.copy_(value.to(local.device)) + return + if value_shape != global_shape: + raise ValueError( + f'Cannot write tensor with shape {value_shape} to distributed parameter with global shape ' + f'{global_shape} and local shape {local_shape}') + from torch.distributed.tensor import distribute_tensor + + value = distribute_tensor(value.to(tensor.device), tensor.device_mesh, tensor.placements) + else: + value = value.to(tensor.device) + tensor.copy_(value) + @staticmethod def synchronize(device=None): """Wait for all queued work on ``device``, defaulting to the LOCAL_RANK device. diff --git a/tests/model/test_megatron_generative_reranker.py b/tests/model/test_megatron_generative_reranker.py new file mode 100644 index 000000000..a13b85f97 --- /dev/null +++ b/tests/model/test_megatron_generative_reranker.py @@ -0,0 +1,20 @@ +from types import SimpleNamespace + +import torch + +from twinkle.model.tasks import _generative_reranker_score + + +def test_generative_reranker_scores_yes_minus_no_and_backpropagates(): + tokenizer = SimpleNamespace(convert_tokens_to_ids=lambda token: {'yes': 3, 'no': 7}[token]) + template = SimpleNamespace(processor=object(), tokenizer=tokenizer) + logits = torch.randn(2, 5, 11, requires_grad=True) + + scores = _generative_reranker_score(logits, template) + + assert scores.shape == (2, 5, 1) + torch.testing.assert_close(scores, logits[..., 3:4] - logits[..., 7:8]) + scores.sum().backward() + assert torch.count_nonzero(logits.grad[..., 3]) == 10 + assert torch.count_nonzero(logits.grad[..., 7]) == 10 + assert torch.count_nonzero(logits.grad[..., [0, 1, 2, 4, 5, 6, 8, 9, 10]]) == 0 diff --git a/tests/model/test_multimodal_lora_slots.py b/tests/model/test_multimodal_lora_slots.py new file mode 100644 index 000000000..7a621e308 --- /dev/null +++ b/tests/model/test_multimodal_lora_slots.py @@ -0,0 +1,85 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import inspect +from copy import deepcopy +from types import SimpleNamespace + +import pytest +import torch +from peft import LoraConfig +from peft.tuners.lora import LoraLayer + +from twinkle.model.multi_lora import MultiLora +from twinkle.model.transformers.multi_lora_transformers import MultiLoraTransformersModel +from twinkle.model.transformers.transformers import TransformersModel + + +class _VisionLanguageModel(torch.nn.Module): + + def __init__(self): + super().__init__() + self.language = torch.nn.Sequential(torch.nn.Linear(4, 4)) + self.visual = torch.nn.Sequential(torch.nn.Linear(4, 4)) + self.aligner = torch.nn.Sequential(torch.nn.Linear(4, 4)) + + def forward(self, x): + return self.language(x) + self.aligner(self.visual(x)) + + +@pytest.mark.parametrize('vision_only', [False, True]) +def test_multimodal_tenant_updates_only_requested_towers(vision_only): + manager = MultiLora(max_loras=2, max_r=2) + module = manager.patch(_VisionLanguageModel(), lora_config=LoraConfig(r=2, target_modules='all-linear')) + manager.save_initial_weights() + manager.reset_adapter_status() + host = SimpleNamespace( + model=module, model_arch={'language_model': ['language'], 'vision_tower': ['visual'], 'aligner': ['aligner']}, + multi_adapter=manager, strategy=SimpleNamespace(unwrap_model=lambda model: model), + _copy_lora_config=MultiLoraTransformersModel._copy_lora_config, + _register_adapter=lambda name, config, **kwargs: manager.acquire_lora(name, config), + ) + host._expand_multimodal_lora_target = lambda *args, **kwargs: TransformersModel._expand_multimodal_lora_target( + host, *args, **kwargs) + config = LoraConfig(r=2, lora_alpha=2, target_modules='all-linear', lora_dropout=0.0) + original = deepcopy(config.to_dict()) + inspect.unwrap(MultiLoraTransformersModel.add_adapter_to_model)( + host, 'tenant', config, freeze_llm=vision_only, freeze_vit=not vision_only) + assert config.to_dict() == original + before = {name: value.detach().clone() for name, value in module.named_parameters()} + with manager.adapter('tenant'): + optimizer = torch.optim.SGD([p for p in module.parameters() if p.requires_grad], lr=0.1) + module(torch.ones(2, 4)).square().sum().backward() + optimizer.step() + changed = [name for name, value in module.named_parameters() if not torch.equal(value, before[name])] + assert changed + expected = '.visual.' if vision_only else '.language.' + assert all(expected in name and '.lora_' in name for name in changed), changed + + +def test_megatron_slots_share_each_chunks_original_target_modules(monkeypatch): + from twinkle.model.megatron.megatron import MegatronModel + + class Chunk(torch.nn.Module): + + def __init__(self, name): + super().__init__() + self.add_module(name, torch.nn.Linear(4, 4)) + self.visual = torch.nn.Linear(4, 4) + + chunks = [Chunk('first'), Chunk('second')] + calls = [] + + def targets(model, requested): + # Expansion must inspect each pristine chunk, before PEFT changes the module tree. + assert not any(isinstance(layer, LoraLayer) for layer in model.modules()) + assert requested == ['all-linear'] + calls.append(model) + return ['first' if model is chunks[0] else 'second'] + + monkeypatch.setattr(MegatronModel, 'get_target_modules', staticmethod(targets)) + manager = MultiLora(max_loras=2, max_r=2) + wrapped = manager.patch(chunks, lora_config=LoraConfig(r=2, target_modules='all-linear')) + assert calls == chunks + for chunk in wrapped: + adapted = [(name, layer) for name, layer in chunk.named_modules() if isinstance(layer, LoraLayer)] + assert len(adapted) == 1 and '.visual' not in adapted[0][0] + assert set(adapted[0][1].lora_A) == {'lora_0', 'lora_1'} diff --git a/tests/server/config/test_training_config.py b/tests/server/config/test_training_config.py index d288bcb4e..61a7686a9 100644 --- a/tests/server/config/test_training_config.py +++ b/tests/server/config/test_training_config.py @@ -31,7 +31,8 @@ def model_application(): @pytest.mark.asyncio -async def test_custom_model_specs_survive_catalog_roundtrip_and_worker_construction(monkeypatch): +@pytest.mark.parametrize('mode', ['lora', 'full']) +async def test_custom_model_specs_survive_catalog_roundtrip_and_worker_construction(monkeypatch, mode): from types import SimpleNamespace from unittest.mock import AsyncMock, Mock from twinkle.server.config.training import TrainingModelConfig @@ -42,7 +43,7 @@ async def test_custom_model_specs_survive_catalog_roundtrip_and_worker_construct app = model_application() # These modules deliberately do not exist on the gateway. Resolution belongs to workers. app['args'].update(model_cls='worker_only.encoder:Model', model_loader='worker_only.loader:load', - gradient_checkpointing=False, mixed_precision='no') + gradient_checkpointing=False, mixed_precision='no', train_mode=mode) catalog = ServerConfig(applications=[app]).training_config spec = TrainingModelConfig.model_validate_json(catalog.models[0].model_dump_json()) captured = Mock(return_value=object()) @@ -55,7 +56,7 @@ async def test_custom_model_specs_survive_catalog_roundtrip_and_worker_construct pool=SimpleNamespace(name=lambda *args: 'owned-instance'), deployment_options=lambda *args: {}, deploy=AsyncMock()) await driver.deploy(InstanceRecord(instance_id='instance', controller_epoch='epoch', - base_model=spec.name, mode='lora', startup_deadline=0), spec) + base_model=spec.name, mode=mode, startup_deadline=0), spec) assert captured.call_args.kwargs['model_cls'] == app['args']['model_cls'] assert captured.call_args.kwargs['model_loader'] == app['args']['model_loader'] assert captured.call_args.kwargs['gradient_checkpointing'] is False @@ -63,13 +64,12 @@ async def test_custom_model_specs_survive_catalog_roundtrip_and_worker_construct driver.runtime.deploy.assert_awaited_once() -@pytest.mark.parametrize('mutation', ['full', 'hybrid', 'megatron']) -def test_custom_model_class_rejects_incompatible_training_modes(mutation): +@pytest.mark.parametrize('mutation,mode', [('hybrid', 'lora'), ('megatron', 'lora'), ('megatron', 'full')]) +def test_custom_model_class_rejects_incompatible_training_modes(mutation, mode): app = model_application() app['args']['model_cls'] = 'worker_only.encoder:Model' - if mutation == 'full': - app['args']['train_mode'] = 'full' - elif mutation == 'hybrid': + app['args']['train_mode'] = mode + if mutation == 'hybrid': app['args']['hybrid'] = {'allocation_path': '/tmp/allocation.json'} else: app['args']['backend'] = 'megatron' @@ -185,18 +185,15 @@ def test_existing_yaml_examples_keep_their_model_entries(monkeypatch): paths += list(Path('cookbook/client/async_rl').glob('server_config.yaml')) paths += list(Path('tests/server/config').glob('server_config*e2e*.yaml')) paths += list(Path('tests/server/fixtures').glob('server_config_mock*.yaml')) - assert len(paths) == 9 + assert paths for path in paths: config = ServerConfig.from_yaml(path) models = [app for app in config.applications if app.import_path == 'model'] - assert len(models) == len(config.training_config.models) == 1 - model = config.training_config.models[0] - assert model.model_id == models[0].args.model_id - assert model.route_prefix == models[0].route_prefix - mode = model.config - original = models[0] - expected_min = original.args.min_replicas - assert mode.min_replicas == expected_min + assert len(models) == len(config.training_config.models) > 0 + for original, model in zip(models, config.training_config.models, strict=True): + assert model.model_id == original.args.model_id + assert model.route_prefix == original.route_prefix + assert model.config.min_replicas == original.args.min_replicas assert ServerConfig.model_validate(config.to_yaml_dict()).training_config == config.training_config @@ -417,3 +414,17 @@ def test_transformers_still_accepts_single_device(): app = model_application() app['args'].update(device_group={'ranks': 1, 'device_type': 'cuda'}, device_mesh={'dp_size': 1}) assert ServerConfig(applications=[app]).training_config.models[0].config.num_devices == 1 + + +@pytest.mark.parametrize('backend', ['transformers', 'megatron']) +@pytest.mark.parametrize('mode', ['lora', 'full']) +def test_native_backends_preserve_loader_and_precision_configuration(backend, mode): + app = model_application() + app['args'].update(backend=backend, train_mode=mode, mixed_precision='no', + model_loader='worker_only.loader:load') + config = ServerConfig(applications=[app]) + restored = ServerConfig.model_validate(config.to_yaml_dict()) + model = restored.training_config.models[0] + assert model.backend == backend and model.mode == mode + assert model.config.model_loader == 'worker_only.loader:load' + assert model.config.mixed_precision == 'no' diff --git a/tests/server/integration/task_http_validation.py b/tests/server/integration/task_http_validation.py index 3f4468b56..dae555fd7 100644 --- a/tests/server/integration/task_http_validation.py +++ b/tests/server/integration/task_http_validation.py @@ -110,9 +110,18 @@ def weights(saved): initial_weights = weights(save(a, 'initial', 0)) for _ in range(3): step(a) + if args.backend == 'transformers': + from twinkle_client.exceptions import TaskFailedError + try: + save(a, 'partial', 3 * len(batch)) + except TaskFailedError as error: + assert 'complete gradient accumulation window' in str(error) + else: + raise AssertionError('An optimizer checkpoint with pending gradients was accepted') + step(a) trained = read(a) np.testing.assert_allclose(read(b), peer, rtol=0, atol=0) - snapshot = save(a, 'trained', 3 * len(batch)) + snapshot = save(a, 'trained', 4 * len(batch)) changed = weights(snapshot) assert changed.keys() == initial_weights.keys() updates = [key for key in changed if not np.array_equal(changed[key].float().numpy(), @@ -130,7 +139,7 @@ def weights(saved): assert fresh_state['cur_step'] == 0 np.testing.assert_allclose(read(c), initial, rtol=0, atol=2e-5) progress = c.resume_from_checkpoint(snapshot.twinkle_path) - assert progress['cur_step'] == 3 and progress['consumed_train_samples'] == 3 * len(batch) + assert progress['cur_step'] == 4 and progress['consumed_train_samples'] == 4 * len(batch) np.testing.assert_allclose(read(c), trained, rtol=0, atol=2e-5) step(c) step(c) diff --git a/tests/server/lifecycle/test_to_backend_inputs.py b/tests/server/lifecycle/test_to_backend_inputs.py index 4be370375..17b44e297 100644 --- a/tests/server/lifecycle/test_to_backend_inputs.py +++ b/tests/server/lifecycle/test_to_backend_inputs.py @@ -12,6 +12,26 @@ from twinkle.data_format import InputFeature, Trajectory from twinkle.server.lifecycle.submit import to_backend_inputs + +@pytest.mark.parametrize('single', [False, True]) +def test_wire_media_restores_tensor_shape_for_swift_collation(single): + import torch + from swift.dev.template import Template + + from twinkle.protocol.types.data import WireInputFeature + + wire = WireInputFeature(input_ids=[1, 2], pixel_values=[[0.5, 0.25], [0.1, 0.2]], + image_grid_thw=[[1, 1, 2]], target_probs=[[0.2, 0.8]]) + output = to_backend_inputs([wire], single=single) + row = output if single else output[0] + media = Template._data_collator_mm_data(object.__new__(Template), [row]) + assert media['pixel_values'].shape == (2, 2) + assert media['image_grid_thw'].shape == (1, 3) + assert media['pixel_values'].dtype == torch.float32 + assert media['image_grid_thw'].dtype == torch.int64 + assert row['target_probs'] == [[0.2, 0.8]] + assert wire.pixel_values == [[0.5, 0.25], [0.1, 0.2]] + _IF = {'input_ids': [1, 2, 3]} _TRAJ = {'messages': [{'role': 'user', 'content': 'hi'}]} diff --git a/tests/server/model/test_task_head.py b/tests/server/model/test_task_head.py index f30808a7e..8c8d8cc5e 100644 --- a/tests/server/model/test_task_head.py +++ b/tests/server/model/test_task_head.py @@ -104,7 +104,8 @@ def tokenizer(text, **kwargs): @pytest.mark.parametrize('task,width', [('seq_cls', 3), ('reranker', 1), ('value', 1)]) -def test_transformers_backend_preserves_validated_head_contract(monkeypatch, task, width): +@pytest.mark.parametrize('mode', ['lora', 'full']) +def test_transformers_backend_preserves_validated_head_contract(monkeypatch, task, width, mode): from transformers import AutoConfig from twinkle.hub import HubOperation from twinkle.server.model import app @@ -114,8 +115,125 @@ def test_transformers_backend_preserves_validated_head_contract(monkeypatch, tas monkeypatch.setattr(AutoConfig, 'from_pretrained', lambda *a, **kw: SimpleNamespace( architectures=['Qwen2ForCausalLM'])) received = {} - monkeypatch.setattr(transformers_model, 'TwinkleCompatTransformersModel', lambda **kwargs: received.update(kwargs)) - kwargs = {'model_id': 'unused', 'task_type': task, 'num_labels': width} + model_class = 'TwinkleCompatFullTransformersModel' if mode == 'full' else 'TwinkleCompatTransformersModel' + monkeypatch.setattr(transformers_model, model_class, lambda **kwargs: received.update(kwargs)) + kwargs = {'model_id': 'unused', 'task_type': task, 'num_labels': width, 'train_mode': mode} app._validate_model_task(kwargs) app._make_transformers_model(kwargs) assert received == {'model_id': 'unused', 'task': task, 'num_labels': width} + + +def test_full_persisted_head_does_not_forward_num_labels_to_hf(monkeypatch): + from twinkle.server.model import app + from twinkle.server.model.backends import transformers_model + + received = {} + monkeypatch.setattr(transformers_model, 'TwinkleCompatFullTransformersModel', + lambda **kwargs: received.update(kwargs)) + app._make_transformers_model({'model_id': 'persisted-classifier', 'train_mode': 'full', + 'num_labels': 3, 'max_loras': 2}) + assert received == {'model_id': 'persisted-classifier'} + + +def test_full_model_attaches_head_and_preserves_device_mesh(monkeypatch): + import inspect + + from twinkle import DeviceMesh + from twinkle.infra.dispatch import _get_device_mesh_param_name + from twinkle.server.model.backends import transformers_model + + mesh = DeviceMesh.from_sizes(dp_size=2, device_type='cpu') + received = {} + monkeypatch.setattr(transformers_model.TransformersModel, '__init__', + lambda self, **kwargs: received.update(kwargs)) + attached = [] + monkeypatch.setattr(transformers_model.TransformersModel, '_add_task_head', + lambda self, *args: attached.append(args)) + model_class = transformers_model.TwinkleCompatFullTransformersModel + inspect.unwrap(model_class.__init__)( + object.__new__(model_class), model_id='causal-backbone', device_mesh=mesh, task='seq_cls', num_labels=3) + assert attached == [('', 'seq_cls', None, 3)] + assert received['device_mesh'] is mesh + assert _get_device_mesh_param_name( + transformers_model.TwinkleCompatFullTransformersModel.__init__) == 'device_mesh' + + +def test_full_custom_model_class_uses_compat_dispatch(monkeypatch): + from twinkle.server.model import app + + marker = type('CustomModel', (), {}) + received = {} + monkeypatch.setattr(app, '_resolve_model_cls', lambda spec: marker) + monkeypatch.setattr(app, '_make_model_class', + lambda base: lambda **kwargs: received.update(base=base, kwargs=kwargs)) + app._make_transformers_model({'model_id': 'st-model', 'train_mode': 'full', + 'model_cls': 'worker_only.encoder:Model'}) + assert received['base'] is marker + assert received['kwargs'] == {'model_id': 'st-model'} + + +def test_embedding_deployment_accepts_explicit_task(monkeypatch): + from transformers import AutoConfig + + from twinkle.hub import HubOperation + + args = TrainingArguments(task_type='embedding') + monkeypatch.setattr(HubOperation, 'download_model', lambda path: path) + monkeypatch.setattr(AutoConfig, 'from_pretrained', lambda *a, **kw: SimpleNamespace( + architectures=['Qwen2ForCausalLM'])) + kwargs = {'model_id': 'causal-backbone', 'task_type': args.task_type} + _validate_model_task(kwargs) + assert kwargs == {'model_id': 'causal-backbone'} + + +def test_decision_requires_a_family_head_factory(): + with pytest.raises(ValueError, match='family model_loader'): + _validate_model_task({'task_type': 'decision', 'model_loader': SimpleNamespace()}) + with pytest.raises(ValidationError, match='model_loader'): + TrainingArguments(task_type='decision') + + +@pytest.mark.parametrize('mode', ['lora', 'full']) +def test_transformers_decision_uses_loader_architecture(monkeypatch, mode): + from twinkle.server.model import app + from twinkle.server.model.backends import transformers_model + + received = {} + for name in ('TwinkleCompatFullTransformersModel', 'TwinkleCompatTransformersModel'): + monkeypatch.setattr(transformers_model, name, lambda **kw: received.update(kw)) + loader = SimpleNamespace(task_type='decision', task_head_loader=lambda: ('family', 'Loader'), + model_arch=SimpleNamespace( + language_model=['model.language_model'], vision_tower=['model.visual'], + aligner=['model.visual.merger'], lm_head='lm_head')) + args = TrainingArguments(task_type='decision', model_loader='family:build') + kwargs = {'model_id': 'decision-backbone', 'train_mode': mode, + 'task_type': args.task_type, 'model_loader': loader} + app._validate_model_task(kwargs) + app._make_transformers_model(kwargs) + assert received['model_arch']['vision_tower'] == ['model.visual'] + if mode == 'full': + assert 'task' not in received and 'task_head_loader' not in received + else: + assert received['task'] == 'decision' + assert received['task_head_loader'] == ('family', 'Loader') + + +@pytest.mark.parametrize('model_cls', [None, 'swift.dev.model.sentence_transformer_model:SentenceTransformerModel']) +def test_native_st_guard_uses_resolved_directory(tmp_path, monkeypatch, model_cls): + from transformers import AutoConfig + from twinkle.hub import HubOperation + + (tmp_path / 'modules.json').write_text('[]') + monkeypatch.setattr(HubOperation, 'download_model', lambda path: str(tmp_path)) + config_reads = [] + monkeypatch.setattr(AutoConfig, 'from_pretrained', + lambda *a, **kw: config_reads.append(a) or SimpleNamespace(architectures=['BertModel'])) + kwargs = {'model_id': 'ms://sentence-transformers/LaBSE', 'task_type': 'embedding'} + if model_cls is not None: + kwargs['model_cls'] = model_cls + _validate_model_task(kwargs) + assert config_reads + else: + with pytest.raises(ValueError, match='native sentence-transformers checkpoint requires model_cls'): + _validate_model_task(kwargs) + assert not config_reads diff --git a/tests/transformers/test_checkpoint_writer_rank.py b/tests/transformers/test_checkpoint_writer_rank.py index 1de64461c..ed62c10d4 100644 --- a/tests/transformers/test_checkpoint_writer_rank.py +++ b/tests/transformers/test_checkpoint_writer_rank.py @@ -20,9 +20,11 @@ def test_full_checkpoint_has_one_writer_after_collectives(self): wrapper.strategy = Mock() wrapper.strategy.unwrap_model.return_value = model wrapper.strategy.get_full_state_dict.return_value = state - wrapper.optimizer_group = {'': SimpleNamespace(cur_step=3)} + wrapper.optimizer_group = {'': SimpleNamespace( + cur_step=3, gradient_accumulation_steps=1, optimizer=None, require_finalized=Mock())} wrapper._save_training_state = Mock() wrapper._save_tokenizer = Mock() + wrapper._save_task_head_checkpoint = Mock() wrapper.submit_async_write = Mock() with patch(f'{module}.Platform.is_master', return_value=rank == 0), \ patch(f'{module}.copy_checkpoint_args'), patch(f'{module}.rotate_checkpoints'): @@ -47,6 +49,19 @@ def test_full_checkpoint_has_one_writer_after_collectives(self): else: model.save_pretrained.assert_not_called() + def test_partial_accumulation_is_rejected_before_weight_collection(self): + wrapper = object.__new__(TransformersModel) + wrapper.strategy = Mock() + wrapper.optimizer_group = {'': SimpleNamespace( + cur_step=3, gradient_accumulation_steps=2, optimizer=None, require_finalized=Mock())} + + with self.assertRaisesRegex(ValueError, 'complete gradient accumulation window'): + inspect.unwrap(TransformersModel.save)( + wrapper, name='partial', output_dir='/unused', save_optimizer=True) + + wrapper.strategy.get_full_state_dict.assert_not_called() + wrapper.strategy.unwrap_model.assert_not_called() + if __name__ == '__main__': unittest.main() diff --git a/tests/transformers/test_dev_task_training.py b/tests/transformers/test_dev_task_training.py index 34f196522..f747477f6 100644 --- a/tests/transformers/test_dev_task_training.py +++ b/tests/transformers/test_dev_task_training.py @@ -101,6 +101,16 @@ def step(model, name): initial, peer = read(model, 'a'), read(model, 'b') for _ in range(3): step(model, 'a') + with pytest.raises(ValueError, match='complete gradient accumulation window'): + model.save('partial', output_dir=str(tmp_path), adapter_name='a', save_optimizer=True) + assert not (tmp_path / 'partial' / 'trainer_state.json').exists() + model.save('weights-only', output_dir=str(tmp_path), adapter_name='a') + assert (tmp_path / 'weights-only' / 'adapter_model.safetensors').exists() + # A complete backward window still needs its optimizer step and zero_grad. + model.forward_backward(inputs=copy.deepcopy(batch), task=task, adapter_name='a') + with pytest.raises(ValueError, match='zero_grad'): + model.save('before-step', output_dir=str(tmp_path), adapter_name='a', save_optimizer=True) + model.clip_grad_and_step(adapter_name='a') trained = read(model, 'a') assert not torch.equal(initial, trained) torch.testing.assert_close(read(model, 'b'), peer, rtol=0, atol=0) @@ -109,7 +119,7 @@ def step(model, name): assert not parameter.requires_grad torch.testing.assert_close(parameter, base[key], rtol=0, atol=0) saved = model.save('trained', output_dir=str(tmp_path), adapter_name='a', - save_optimizer=True, consumed_train_samples=3 * len(batch)) + save_optimizer=True, consumed_train_samples=4 * len(batch)) if task != 'embedding': head = {k: v for k, v in load_file(saved + '/adapter_model.safetensors').items() if 'score.' in k} assert head and any(torch.count_nonzero(v) for k, v in head.items() if 'lora_B' in k) @@ -128,7 +138,7 @@ def step(model, name): cold = build() add(cold, 'a') progress = cold.resume_from_checkpoint(saved, adapter_name='a') - assert progress['cur_step'] == 3 + assert progress['cur_step'] == 4 step(cold, 'a') step(cold, 'a') torch.testing.assert_close(read(cold, 'a'), continued, rtol=0, atol=1e-6) diff --git a/tests/transformers/test_full_checkpoint_resume.py b/tests/transformers/test_full_checkpoint_resume.py index 1f44a4ad1..d8b7cb530 100644 --- a/tests/transformers/test_full_checkpoint_resume.py +++ b/tests/transformers/test_full_checkpoint_resume.py @@ -21,7 +21,7 @@ def test_full_resume_restores_weights_and_optional_training_state(tmp_path, weig torch.save(saved_optimizer.state_dict(), tmp_path / 'optimizer.pt') save_file({'weight': torch.full((1, 1), 2.)}, str(tmp_path / 'model.safetensors')) (tmp_path / 'trainer_state.json').write_text(json.dumps({ - 'cur_step': 3, 'consumed_train_samples': 12, 'gradient_accumulation_steps': 2, + 'cur_step': 4, 'consumed_train_samples': 12, 'gradient_accumulation_steps': 2, })) wrapper = object.__new__(TransformersModel) wrapper.__dict__['model'] = model @@ -31,14 +31,14 @@ def test_full_resume_restores_weights_and_optional_training_state(tmp_path, weig result = wrapper.resume_from_checkpoint(str(tmp_path), resume_only_model=weights_only) assert model(torch.ones(1, 1)).item() == 2 # dev's weights-only contract starts fresh progress as well as fresh optimizer state. - assert result['cur_step'] == (0 if weights_only else 3) + assert result['cur_step'] == (0 if weights_only else 4) assert result['consumed_train_samples'] == (0 if weights_only else 12) if weights_only: assert not optimizer.state assert wrapper.optimizer_group[''].cur_step == 0 else: torch.testing.assert_close(optimizer.state[model.weight]['exp_avg'], torch.full((1, 1), 0.5)) - assert wrapper.optimizer_group[''].cur_step == 3 + assert wrapper.optimizer_group[''].cur_step == 4 assert wrapper.optimizer_group[''].gradient_accumulation_steps == 2 @@ -166,3 +166,12 @@ def test_rebind_is_a_noop_when_the_optimizer_is_already_bound(): group = SimpleNamespace(lr_scheduler=scheduler, _scheduler_spec=None) TransformersModel._rebind_lr_scheduler(group, built_on) # same optimizer object assert scheduler.base_lrs == [0.05] + + +def test_optimizer_resume_rejects_legacy_partial_accumulation_checkpoint(tmp_path): + wrapper = object.__new__(TransformersModel) + (tmp_path / 'trainer_state.json').write_text(json.dumps({ + 'cur_step': 3, 'gradient_accumulation_steps': 2, + })) + with pytest.raises(ValueError, match='saved during gradient accumulation'): + wrapper._restore_training_state(str(tmp_path)) diff --git a/tests/transformers/test_megatron_weight_only_resume.py b/tests/transformers/test_megatron_weight_only_resume.py index 1fda7208d..16cc9299c 100644 --- a/tests/transformers/test_megatron_weight_only_resume.py +++ b/tests/transformers/test_megatron_weight_only_resume.py @@ -1,3 +1,4 @@ +import sys from contextlib import nullcontext from copy import deepcopy from types import SimpleNamespace @@ -40,12 +41,15 @@ def test_weights_only_load_refreshes_target_master_params_and_preserves_adam_sta def load_weights(*args, **kwargs): parameter.data.fill_(2.) + bridge = SimpleNamespace( + load_weights=load_weights, _convert_post_process=lambda *args: None, _peft_format=False, + hf_score_key='score.weight', config=SimpleNamespace(task_type='causal_lm')) wrapper = SimpleNamespace( _get_default_group=lambda: 'target', model=[parameter], - strategy=SimpleNamespace(unwrap_model=lambda model: model, - bridge=SimpleNamespace(load_weights=load_weights)), + strategy=SimpleNamespace(unwrap_model=lambda model: model, bridge=bridge), optimizer_group={'target': group, 'other': SimpleNamespace(optimizer=other)}, multi_adapter=SimpleNamespace(save_context=lambda adapter: nullcontext('slot'), load_lora_converter=None), + _load_task_head_checkpoint=lambda *args, **kwargs: None, ) model_class.load(wrapper, str(tmp_path), adapter_name='target', load_optimizer=False) assert parameter.item() == 2. @@ -53,3 +57,102 @@ def load_weights(*args, **kwargs): optimizer.step() torch.testing.assert_close(parameter, expected.model_parameter) assert other.master.item() == other.model_parameter.item() == 11. + + +@pytest.mark.parametrize('api', ['legacy', 'current']) +def test_mcore_optimizer_save_uses_versioned_torch_dist_strategy(tmp_path, monkeypatch, api): + pytest.importorskip('megatron.core', reason='requires the optional Megatron backend') + from megatron.core import dist_checkpointing + from megatron.core import parallel_state as mpu + + captured = {} + + def record(strategy, async_save, **kwargs): + captured.update(strategy=strategy, async_save=async_save, **kwargs) + + def legacy_save(state_dict, checkpoint_dir, sharded_strategy=None, common_strategy=None, + validate_access_integrity=True, async_sharded_save=False, + preprocess_common_before_consistancy_check=None): + record(sharded_strategy, async_sharded_save) + + def current_save(state_dict, checkpoint_dir, sharded_strategy=None, common_strategy=None, + validate_access_integrity=True, async_sharded_save=False, + preprocess_common_before_consistancy_check=None, content_metadata=None, + async_strategy='nvrx'): + record(sharded_strategy, async_sharded_save, metadata=content_metadata, async_strategy=async_strategy) + + monkeypatch.setattr(dist_checkpointing, 'save', legacy_save if api == 'legacy' else current_save) + monkeypatch.setattr(mpu, 'get_data_parallel_world_size', lambda **kwargs: 1) + metadata = {'sharded': True, 'distrib_optim_sharding_type': 'dp_reshardable'} + wrapper = SimpleNamespace( + model=object(), + strategy=SimpleNamespace(get_sharded_sd_metadata=lambda: metadata), + _generate_state_dict=lambda **kwargs: {'optimizer': {}}, + ) + optimizer = SimpleNamespace(cur_step=1, optimizer=object(), lr_scheduler=None) + + MegatronModel._save_mcore_optimizer( + wrapper, str(tmp_path), optimizer, no_save_rng=True, thread_count=3, async_save=True) + + assert captured['strategy'].backend == 'torch_dist' + assert captured['strategy'].version == 1 + assert captured['strategy'].thread_count == 3 + assert captured['async_save'] is False + if api == 'current': + assert captured['metadata'] == {'sharded': True, 'distrib_optim_sharding_type': 'dp_reshardable'} + assert captured['async_strategy'] == 'mcore' + else: + assert 'metadata' not in captured + assert 'async_strategy' not in captured + assert (tmp_path / 'latest_checkpointed_iteration.txt').read_text() == '1' + + +def test_mixed_precision_optimizer_checkpoint_uses_the_actual_main_param_order(): + pytest.importorskip('megatron.core', reason='requires the optional Megatron backend') + from megatron.core.optimizer.distrib_optimizer import DistributedOptimizer + + from twinkle.model.megatron.megatron import _align_distributed_optimizer_param_order + + low = torch.nn.Parameter(torch.ones(8, dtype=torch.bfloat16)) + head = torch.nn.Parameter(torch.ones(3)) + master = torch.nn.Parameter(low.float().clone()) + distributed = object.__new__(DistributedOptimizer) + distributed.model_float16_groups = [[low]] + distributed.model_fp32_groups = [[head]] + distributed.model_param_group_index_map = {low: (0, 0), head: (0, 1)} + distributed.optimizer = torch.optim.Adam([head, master]) + distributed.ddp_config = SimpleNamespace(use_custom_fsdp=False) + distributed.config = SimpleNamespace( + use_precision_aware_optimizer=False, use_precision_aware_optimizer_no_fp8_or_ds_fp8=False) + + _align_distributed_optimizer_param_order(SimpleNamespace(chained_optimizers=[distributed])) + + for param, expected in [(low, master), (head, head)]: + assert distributed._get_main_param_and_optimizer_states(param)['param'] is expected + + +def test_mcore_persisted_classifier_config(monkeypatch): + pytest.importorskip('megatron.core', reason='requires the optional Megatron backend') + from twinkle import Platform + from twinkle.model.megatron.bridge.mcore import MCoreBridgeBackend + + bridge = SimpleNamespace( + ModelConfig=lambda **kwargs: kwargs, + hf_to_mcore_config=lambda config: {'untie_embeddings_and_output_weights': False}, + ) + monkeypatch.setitem(sys.modules, 'mcore_bridge', bridge) + monkeypatch.setattr(Platform, 'get_platform', lambda: SimpleNamespace( + configure_model_config=lambda config: None)) + strategy = SimpleNamespace(params_type='float32', sequence_parallel=False, variable_seq_lengths=False) + hf_config = SimpleNamespace(architectures=['Qwen2ForSequenceClassification'], num_labels=3) + + config = MCoreBridgeBackend().build_model_config( + hf_config, {}, strategy, untie_embeddings_and_output_weights=False) + + assert config['task_type'] == 'seq_cls' + assert config['num_labels'] == 3 + assert config['untie_embeddings_and_output_weights'] is True + with pytest.raises(ValueError, match='task_type'): + MCoreBridgeBackend().build_model_config(hf_config, {}, strategy, task_type='causal_lm') + with pytest.raises(ValueError, match='num_labels'): + MCoreBridgeBackend().build_model_config(hf_config, {}, strategy, num_labels=2) diff --git a/tests/transformers/test_model_loader_construction.py b/tests/transformers/test_model_loader_construction.py new file mode 100644 index 000000000..572ab0f30 --- /dev/null +++ b/tests/transformers/test_model_loader_construction.py @@ -0,0 +1,63 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""A supplied loader owns external architectures, including empty-weight construction.""" +import pytest +import torch + + +@pytest.mark.parametrize('multi', [False, True]) +def test_loader_builds_architecture_outside_transformers(tmp_path, monkeypatch, multi): + from tokenizers import Tokenizer + from tokenizers.models import WordPiece + from transformers import PreTrainedTokenizerFast, Qwen2Config, Qwen2ForCausalLM + + from twinkle import Platform, initialize + from twinkle.model.transformers.multi_lora_transformers import MultiLoraTransformersModel + from twinkle.model.transformers.transformers import TransformersModel + + monkeypatch.setenv('ACCELERATE_USE_CPU', 'true') + monkeypatch.setattr(Platform, 'get_local_device', lambda *args, **kwargs: 'cpu') + initialize(mode='local') + config = Qwen2Config(vocab_size=32, hidden_size=16, intermediate_size=32, num_hidden_layers=1, + num_attention_heads=2, num_key_value_heads=2, pad_token_id=0) + config.architectures = ['ExternalArchitectureOwnedByLoader'] + config.save_pretrained(tmp_path) + tokenizer = PreTrainedTokenizerFast( + tokenizer_object=Tokenizer(WordPiece({'[PAD]': 0, '[UNK]': 1}, unk_token='[UNK]')), + pad_token='[PAD]', unk_token='[UNK]') + tokenizer.save_pretrained(tmp_path) + calls = [] + + class Loader: + def build_config(self, path): + return config + + def process_config(self, value): + return value + + def build_processor(self, path, value): + calls.append('processor') + return tokenizer + + def process_tokenizer(self, value): + return value + + def build_model(self, path, **kwargs): + calls.append('model') + return Qwen2ForCausalLM(kwargs['config']) + + def build_model_from_config(self, value, **kwargs): + calls.append('meta') + return Qwen2ForCausalLM(value) + + def process_model(self, value): + return value + + loader = Loader() + cls = MultiLoraTransformersModel if multi else TransformersModel + wrapper = cls(model_id=str(tmp_path), model_loader=loader, mixed_precision='no', + gradient_checkpointing=False, **({'max_loras': 2, 'max_r': 4, + 'target_modules': ['q_proj']} if multi else {})) + assert calls == ['processor', 'model'] + empty = wrapper._init_empty_model_from_config(None, model_loader=loader, processor=tokenizer) + assert all(p.device.type == 'meta' for p in empty.parameters()) + assert calls[-1] == 'meta' diff --git a/tests/transformers/test_native_fsdp_strategy.py b/tests/transformers/test_native_fsdp_strategy.py index 63f8f0e8c..68fa04447 100644 --- a/tests/transformers/test_native_fsdp_strategy.py +++ b/tests/transformers/test_native_fsdp_strategy.py @@ -1,13 +1,14 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from unittest.mock import MagicMock, patch -from types import SimpleNamespace import os +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import pytest import torch -from twinkle.model.transformers.strategy.native_fsdp import NativeFSDPStrategy from twinkle.model.transformers.strategy import native_fsdp +from twinkle.model.transformers.strategy.native_fsdp import NativeFSDPStrategy from twinkle.model.transformers.transformers import TransformersModel @@ -91,6 +92,7 @@ def test_lazy_wrapping_rebinds_each_tenants_optimizer_without_other_parameters(t mesh.mesh_dim_names = None strategy = NativeFSDPStrategy(device_mesh=mesh, enable_ep=False) wrapper = SimpleNamespace(model=original, optimizer_group=groups, strategy=strategy, + model_arch={'language_model': [], 'vision_tower': [], 'aligner': []}, _model_wrapped=False, _enable_expert_parallel=False, sp_strategy=None, _maybe_apply_expert_parallel=lambda: None, _ensure_sp_strategy=lambda: None, _ensure_gdn_padding_free_patch=lambda: None, @@ -109,3 +111,121 @@ def test_lazy_wrapping_rebinds_each_tenants_optimizer_without_other_parameters(t torch.testing.assert_close(replacement['base'], torch.ones(2)) if tenant_count == 1: torch.testing.assert_close(replacement['b'], torch.ones(2)) + + +@pytest.fixture +def head_sharding(): + strategy = NativeFSDPStrategy(device_mesh=MagicMock(), mixed_precision='bf16', enable_ep=False) + with patch('twinkle.model.transformers.strategy.native_fsdp._build_fsdp_mesh', + return_value=MagicMock()), \ + patch('twinkle.model.transformers.strategy.native_fsdp.fully_shard') as fully_shard: + yield strategy, fully_shard + + +def test_builtin_score_head_remains_in_root_fsdp_group(head_sharding): + model = torch.nn.Module() + model.backbone = torch.nn.Linear(4, 4).to(torch.bfloat16) + model.score = torch.nn.Linear(4, 2).to(torch.bfloat16) + strategy, fully_shard = head_sharding + strategy.wrap_model(model) + + fully_shard.assert_called_once() + assert fully_shard.call_args.args[0] is model + assert fully_shard.call_args.kwargs['mp_policy'].param_dtype is torch.bfloat16 + + +def test_nonmaster_save_ignores_empty_gathered_optimizer_state(tmp_path): + strategy = NativeFSDPStrategy(device_mesh=MagicMock(), enable_ep=False) + model = torch.nn.Linear(3, 3) + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + + with patch('twinkle.model.transformers.strategy.native_fsdp.Platform.is_master', return_value=False), \ + patch('torch.distributed.checkpoint.state_dict.get_optimizer_state_dict', return_value={}) as get_state, \ + patch('torch.save') as save: + strategy.save_optimizer_checkpoint(model, optimizer, str(tmp_path / 'optimizer.pt')) + + get_state.assert_called_once() + save.assert_not_called() + + +def test_optimizer_restore_accepts_lazy_state_for_unused_parameters(tmp_path): + strategy = NativeFSDPStrategy(device_mesh=MagicMock(), enable_ep=False) + model = torch.nn.Sequential(torch.nn.Linear(3, 3), torch.nn.Linear(3, 1)) + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + model[0](torch.ones(2, 3)).sum().backward() + optimizer.step() + optimizer.zero_grad() + with patch('twinkle.model.transformers.strategy.native_fsdp.Platform.is_master', return_value=True): + strategy.save_optimizer_checkpoint(model, optimizer, str(tmp_path / 'optimizer.pt')) + state = torch.load(tmp_path / 'optimizer.pt', weights_only=True) + assert state['state']['1.weight'] == {} + + cold = torch.nn.Sequential(torch.nn.Linear(3, 3), torch.nn.Linear(3, 1)) + cold_optimizer = torch.optim.Adam(cold.parameters(), lr=1e-3) + prepare_options = strategy._prepare_optimizer_state_dict_options + + def cpu_options(*, for_load): + options = prepare_options(for_load=for_load) + options.broadcast_from_rank0 = False + return options + + with patch.object(strategy, '_prepare_optimizer_state_dict_options', side_effect=cpu_options), \ + patch('twinkle.model.transformers.strategy.native_fsdp.Platform.is_master', return_value=True): + strategy.load_optimizer_checkpoint(cold, cold_optimizer, str(tmp_path / 'optimizer.pt')) + assert cold_optimizer.state[cold[0].weight]['step'].item() == 1 + assert not cold_optimizer.state[cold[1].weight] + + corrupted = deepcopy(state) + del corrupted['state']['0.weight'] + torch.save(corrupted, tmp_path / 'corrupted.pt') + with patch.object(strategy, '_prepare_optimizer_state_dict_options', side_effect=cpu_options), \ + patch('twinkle.model.transformers.strategy.native_fsdp.Platform.is_master', return_value=True), \ + pytest.raises((KeyError, RuntimeError), match='0.weight'): + strategy.load_optimizer_checkpoint(cold, cold_optimizer, str(tmp_path / 'corrupted.pt')) + + +def test_nonmaster_restores_lazy_slots_from_broadcast_metadata(): + strategy = NativeFSDPStrategy(device_mesh=MagicMock(), enable_ep=False) + model = torch.nn.Sequential(torch.nn.Linear(3, 3), torch.nn.Linear(3, 1)) + optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) + model(torch.ones(2, 3)).sum().backward() + optimizer.step() + optimizer.zero_grad() + + def broadcast_empty_states(payload, src): + assert src == 0 + payload[0] = {'1.weight'} + + with patch('twinkle.model.transformers.strategy.native_fsdp.Platform.is_master', return_value=False), \ + patch('twinkle.model.transformers.strategy.native_fsdp.dist.is_initialized', return_value=True), \ + patch('twinkle.model.transformers.strategy.native_fsdp.dist.broadcast_object_list', + side_effect=broadcast_empty_states), \ + patch('torch.distributed.checkpoint.state_dict.set_optimizer_state_dict') as set_state: + strategy.load_optimizer_checkpoint(model, optimizer, '/unused') + + set_state.assert_called_once() + assert optimizer.state[model[0].weight] + assert not optimizer.state[model[1].weight] + + +def test_fp32_task_head_is_sharded_as_separate_precision_group(head_sharding): + model = torch.nn.Module() + model.backbone = torch.nn.Linear(4, 4).to(torch.bfloat16) + model.task_head = torch.nn.Linear(4, 1) + model.task_head.keep_fp32 = True + model.scoring_head = model.task_head + model.standard_head = torch.nn.Linear(4, 1).to(torch.bfloat16) + model.task_heads = {'': model.standard_head} + strategy, fully_shard = head_sharding + strategy.wrap_model(model) + + assert fully_shard.call_count == 2 + head_call, model_call = fully_shard.call_args_list + assert head_call.args[0] is model.task_head + assert head_call.kwargs['mp_policy'].param_dtype is None + assert head_call.kwargs['mp_policy'].reduce_dtype is None + assert model_call.args[0] is model + assert model_call.kwargs['mp_policy'].param_dtype is torch.bfloat16 + assert model.task_head.bias.dtype is torch.float32 + assert model.standard_head.weight.dtype is torch.bfloat16 + diff --git a/tests/transformers/test_task_head_buffers.py b/tests/transformers/test_task_head_buffers.py new file mode 100644 index 000000000..b414aed01 --- /dev/null +++ b/tests/transformers/test_task_head_buffers.py @@ -0,0 +1,156 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Calibration buffers must follow the tenant through reuse and checkpoint restore.""" +from types import SimpleNamespace + +import pytest +import torch +from torch import nn + +from twinkle.model.base import TrainableModel +from twinkle.model.multi_lora import MultiLoraTaskHeadSlots + + +def _head(): + head = nn.Linear(2, 2) + head.register_buffer('temperature', torch.tensor([1., 2.]), persistent=False) + return head + + +def _write(tensor, value): + tensor.copy_(value.to(tensor)) + + +def test_released_head_restores_nonpersistent_calibration(): + slots, host = MultiLoraTaskHeadSlots(2), nn.Module() + slots.build(host, _head) + slots.bind('a', host) + slots.bind('b', host) + a, b = host.task_heads['a'], host.task_heads['b'] + temperature = a.temperature + a.temperature.fill_(9) + slots.release('a', host, _write) + slots.bind('c', host) + assert host.task_heads['c'].temperature is temperature + torch.testing.assert_close(temperature, torch.tensor([1., 2.])) + torch.testing.assert_close(b.temperature, torch.tensor([1., 2.])) + + +def test_calibrated_checkpoint_moves_to_another_slot(): + slots, host = MultiLoraTaskHeadSlots(2), nn.Module() + slots.build(host, _head) + slots.bind('a', host) + host.task_heads['a'].temperature.fill_(7) + saved = slots.export_state('a', host, lambda tensor: tensor) + slots.release('a', host, _write) + slots.bind('b', host) + slots.bind('a', host) + assert slots.bindings['a'] == 1 + slots.import_state('a', host, saved, _write) + torch.testing.assert_close(host.task_heads['a'].temperature, torch.full((2,), 7.)) + torch.testing.assert_close(host.task_heads['b'].temperature, torch.tensor([1., 2.])) + + +def test_single_head_checkpoint_restores_calibration(tmp_path): + head = _head() + wrapper = SimpleNamespace(_attached_head=lambda name: head, _task_head_is_master=lambda: True, + _task_head_carries_on_this_rank=lambda: True) + for name in ('_task_head_state', '_restore_task_head_state'): + setattr(wrapper, name, getattr(TrainableModel, name).__get__(wrapper)) + head.temperature.fill_(5) + TrainableModel._save_task_head_checkpoint(wrapper, str(tmp_path), None) + head.temperature.zero_() + TrainableModel._load_task_head_checkpoint(wrapper, str(tmp_path), None) + torch.testing.assert_close(head.temperature, torch.full((2,), 5.)) + + +def test_bad_buffer_shape_rejected_before_any_tensor_changes(): + slots, host = MultiLoraTaskHeadSlots(1), nn.Module() + slots.build(host, _head) + slots.bind('a', host) + head = host.task_heads['a'] + before = head.weight.detach().clone() + state = slots.export_state('a', host, lambda tensor: tensor) + state['task_head.weight'].fill_(99) + state['task_head.temperature'] = torch.ones(3) + with pytest.raises(ValueError, match='shape'): + slots.import_state('a', host, state, _write) + torch.testing.assert_close(head.weight, before) + + +@pytest.fixture(params=[False, True], ids=['plain', 'sharded']) +def head_model(tmp_path, request): + from torch.distributed.device_mesh import init_device_mesh + from torch.distributed.tensor import Shard, distribute_tensor + from twinkle.model.transformers.strategy.native_fsdp import NativeFSDPStrategy + from twinkle.model.transformers.transformers import TransformersModel + + sharded = request.param + if sharded: + torch.distributed.init_process_group( + 'gloo', init_method=f'file://{tmp_path / "head-pg"}', rank=0, world_size=1) + try: + mesh = init_device_mesh('cpu', (1,)) if sharded else None + head = nn.Linear(2, 1) + if sharded: + for name, parameter in list(head.named_parameters()): + setattr(head, name, nn.Parameter(distribute_tensor(parameter.detach(), mesh, [Shard(0)]))) + host = nn.Module() + host.task_head = head + model = object.__new__(TransformersModel) + model.__dict__['model'] = host + model.strategy = NativeFSDPStrategy(device_mesh=mesh, enable_ep=False) + yield model, head + finally: + if sharded: + torch.distributed.destroy_process_group() + + +@pytest.mark.parametrize('persistent', [False, True]) +def test_head_checkpoint_restores_buffers_and_keeps_parameters(head_model, persistent): + from torch.distributed.tensor import DTensor + + model, head = head_model + head.register_buffer('calibration', torch.tensor([7.0]), persistent=persistent) + parameters = dict(head.named_parameters()) + state = model._task_head_state('') + assert set(state) == {'task_head.weight', 'task_head.bias', 'task_head.calibration'} + assert all(not isinstance(value, DTensor) and value.device.type == 'cpu' for value in state.values()) + with torch.no_grad(): + for parameter in head.parameters(): + parameter.zero_() + head.calibration.zero_() + model._restore_task_head_state('', state) + for name, parameter in head.named_parameters(): + assert parameter is parameters[name] + value = parameter.full_tensor() if isinstance(parameter, DTensor) else parameter + torch.testing.assert_close(value, state[f'task_head.{name}']) + torch.testing.assert_close(head.calibration, torch.tensor([7.0])) + + +@pytest.mark.parametrize('persistent', [False, True]) +def test_head_loads_legacy_parameters_without_changing_calibration(head_model, persistent): + from torch.distributed.tensor import DTensor + + model, head = head_model + head.register_buffer('calibration', torch.tensor([7.0]), persistent=persistent) + legacy = {key: torch.full_like(value, 3.0) for key, value in model._task_head_state('').items() + if key != 'task_head.calibration'} + model._restore_task_head_state('', legacy) + for parameter in head.parameters(): + value = parameter.full_tensor() if isinstance(parameter, DTensor) else parameter + torch.testing.assert_close(value, torch.full_like(value, 3.0)) + torch.testing.assert_close(head.calibration, torch.tensor([7.0])) + + +def test_head_rejects_bad_buffer_shape_before_loading_parameters(head_model): + model, head = head_model + head.register_buffer('calibration', torch.tensor([7.0]), persistent=False) + before = model._task_head_state('') + state = {key: value.clone() for key, value in before.items()} + state['task_head.weight'].fill_(99) + state['task_head.calibration'] = torch.ones(3) + with pytest.raises(ValueError, match='shape'): + model._restore_task_head_state('', state) + after = model._task_head_state('') + for key, value in before.items(): + torch.testing.assert_close(after[key], value) diff --git a/tests/utils/test_response_token_scores.py b/tests/utils/test_response_token_scores.py new file mode 100644 index 000000000..0cd8103ec --- /dev/null +++ b/tests/utils/test_response_token_scores.py @@ -0,0 +1,24 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Response positions must survive variable-length Megatron microbatches.""" +import pytest +import torch + +from twinkle.signals.response import response_token_scores + + +@pytest.mark.parametrize('serialized', [False, True]) +def test_variable_length_microbatches_keep_feature_order(serialized): + batches = [torch.tensor([[1., 2.], [3., 4.]]), torch.tensor([[5., 6., 7.]])] + values = [batch.tolist() for batch in batches] if serialized else batches + features = [{'labels': [-100, 1]}, {'labels': [1, -100]}, {'labels': [-100, 1, 1]}] + assert response_token_scores({'logps': values}, features) == [[2.], [3.], [6., 7.]] + + +@pytest.mark.parametrize('values', [torch.tensor([[1., 2.], [3., 4.]]), [[1., 2.], [3., 4.]]]) +def test_dense_rows_remain_aligned(values): + assert response_token_scores({'logps': values}, [{'labels': [-100, 1]}] * 2) == [[2.], [4.]] + + +def test_microbatch_row_count_mismatch_fails(): + with pytest.raises(RuntimeError, match='per-row alignment'): + response_token_scores({'logps': [[[1., 2.], [3., 4.]]]}, [{'labels': [-100, 1]}])