From e41008d23702c6505fd6f049522fea7bfd770597 Mon Sep 17 00:00:00 2001 From: QiyuanChen <72334646+qychen2001@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:17:21 +0800 Subject: [PATCH 1/4] perf(transformers): skip unit-temperature logit scaling Co-authored-by: Codex --- src/twinkle/model/transformers/transformers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/twinkle/model/transformers/transformers.py b/src/twinkle/model/transformers/transformers.py index a147f4da2..cb0dedcc9 100644 --- a/src/twinkle/model/transformers/transformers.py +++ b/src/twinkle/model/transformers/transformers.py @@ -911,7 +911,8 @@ def _postprocess_forward_outputs(self, outputs, *, labels, task, temperature, sa loss_mask = (labels != -100).bool() masked_labels = labels.masked_fill(~loss_mask, 0) logits = outputs['logits'] - logits.div_(temperature) + if temperature != 1.0: + logits.div_(temperature) if loss_require_entropy: outputs['logps'], outputs['entropies'] = selective_log_softmax( logits, masked_labels, return_entropy=True) From 1298952d97bd730df07681bb6a65a30f8e07a2ed Mon Sep 17 00:00:00 2001 From: QiyuanChen <72334646+qychen2001@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:17:22 +0800 Subject: [PATCH 2/4] perf(checkpoint): broadcast only valid NCCL bucket bytes Co-authored-by: Codex --- .../nccl_checkpoint_engine.py | 18 ++++--- .../test_nccl_bucket_payload.py | 53 +++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) create mode 100644 tests/checkpoint_engine/test_nccl_bucket_payload.py diff --git a/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py b/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py index 3490f632b..8a0b9c630 100644 --- a/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py +++ b/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py @@ -104,8 +104,10 @@ def _run(self): # A stale wire-probe from the subscription rendezvous: drain it and keep waiting for # the manifest topic (the slow-joiner re-probe leaves extra probes in a fast receiver). - # Broadcast tensor data via NCCL - _pg_broadcast(self.pg, self.bucket, src=0) + # An empty final bucket still sends its manifest to terminate the stream. + length = self.metadata['length'] + if length: + _pg_broadcast(self.pg, self.bucket[:length], src=0) async def wait_for_complete(self) -> dict[str, TensorMeta]: """Wait for the broadcast operation to complete. @@ -457,7 +459,8 @@ async def send_weights( bucket=send_buf, metadata={ 'bucket_meta': bucket_meta, - 'is_last': False + 'is_last': False, + 'length': offset, }, socket=self.socket, topic=self.topic, @@ -496,7 +499,8 @@ async def send_weights( bucket=send_buf, metadata={ 'bucket_meta': bucket_meta, - 'is_last': True + 'is_last': True, + 'length': offset, }, socket=self.socket, topic=self.topic, @@ -536,7 +540,7 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No auth_key=self.auth_key, ) metadata = await broadcast_op.wait_for_complete() - total_bytes += self.bucket_size + total_bytes += metadata['length'] total_params += len(metadata['bucket_meta']) # Swap buffers @@ -563,7 +567,7 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No # 3. Wait for next bucket metadata = await broadcast_op.wait_for_complete() - total_bytes += self.bucket_size + total_bytes += metadata['length'] total_params += len(metadata['bucket_meta']) # 4. Swap buffers @@ -580,5 +584,5 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No elapsed = time.time() - start_time bandwidth = total_bytes / elapsed / (1024 * 1024 * 1024) logger.info(f'receive_weights done: rank={self.rank}, ' - f'params={total_params}, ' + f'params={total_params}, bytes={total_bytes}, ' f'time={elapsed:.2f}s, bandwidth={bandwidth:.2f} GB/s') diff --git a/tests/checkpoint_engine/test_nccl_bucket_payload.py b/tests/checkpoint_engine/test_nccl_bucket_payload.py new file mode 100644 index 000000000..695d8cc4d --- /dev/null +++ b/tests/checkpoint_engine/test_nccl_bucket_payload.py @@ -0,0 +1,53 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Exercise the full-sync bucket protocol with CPU buffers.""" + +import asyncio +from collections import deque +from types import SimpleNamespace + +import pytest +import torch + +from twinkle.checkpoint_engine import nccl_checkpoint_engine as nccl + + +@pytest.mark.parametrize('sizes, broadcast_sizes', [([], []), ([0], []), ([4], [16]), ([3, 2, 3], [12, 8, 12])]) +def test_nccl_broadcast_uses_payload_length(monkeypatch, sizes, broadcast_sizes): + manifests, payloads = deque(), deque() + sent_sizes, received_sizes = [], [] + monkeypatch.setattr(torch.cuda, 'synchronize', lambda: None) + monkeypatch.setattr(nccl, 'send_auth_pyobj', lambda socket, metadata, *args, **kwargs: manifests.append(metadata)) + monkeypatch.setattr(nccl, 'recv_auth_pyobj', lambda *args, **kwargs: manifests.popleft()) + + def broadcast(rank, bucket, src): + if rank == 0: + sent_sizes.append(bucket.numel()) + payloads.append(bucket.clone()) + else: + received_sizes.append(bucket.numel()) + payload = payloads.popleft() + assert payload.shape == bucket.shape + bucket.copy_(payload) + + monkeypatch.setattr(nccl, '_pg_broadcast', broadcast) + + def engine(rank): + engine = nccl.NCCLCheckpointEngine(bucket_size=16) + engine.rank = engine._pg = rank + engine.send_buf = torch.empty(16, dtype=torch.uint8) + engine.recv_buf = torch.empty_like(engine.send_buf) + engine.socket = SimpleNamespace(send_string=lambda *args, **kwargs: None, recv_string=lambda: engine.topic) + return engine + + weights = {str(i): torch.arange(n, dtype=torch.float32) + i for i, n in enumerate(sizes)} + + async def roundtrip(): + await engine(0).send_weights(iter(weights.items())) + return {k: v.clone() async for k, v in engine(1).receive_weights()} + + received = asyncio.run(roundtrip()) + assert sent_sizes == received_sizes == broadcast_sizes + assert received.keys() == weights.keys() + for name in weights: + torch.testing.assert_close(received[name], weights[name], rtol=0, atol=0) + assert not manifests and not payloads From ab7c7531baaa8d3a3775e30b645cace6f5083d36 Mon Sep 17 00:00:00 2001 From: QiyuanChen <72334646+qychen2001@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:24:31 +0800 Subject: [PATCH 3/4] perf: skip remaining unit-temperature tensor scaling Co-authored-by: Codex --- src/twinkle/loss/gkd.py | 21 +++++++++------------ src/twinkle/loss/grpo.py | 2 +- src/twinkle/loss/infonce.py | 16 ++++++++++++---- src/twinkle/loss/reranker.py | 4 +++- src/twinkle/model/megatron/megatron.py | 3 ++- src/twinkle/utils/torch_utils.py | 7 +++++-- 6 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/twinkle/loss/gkd.py b/src/twinkle/loss/gkd.py index 49e48366b..95f16b7f4 100644 --- a/src/twinkle/loss/gkd.py +++ b/src/twinkle/loss/gkd.py @@ -170,24 +170,17 @@ def _generalized_jsd_loss( import torch.nn.functional as F # ── Top-k reduction ────────────────────────────────────────────────── - # Top-k mode: gather/topk first to get small [*, k] tensors, then scale in-place if teacher_topk_logprobs is not None and teacher_topk_indices is not None: # Remote API teacher: teacher already provides top-k log-probs (T=1). - # Gather student logits at teacher's top-k indices, then scale in-place. teacher_topk_indices = teacher_topk_indices.to(student_logits.device) teacher_topk_logprobs = teacher_topk_logprobs.to(student_logits.device) student_logits = torch.gather(student_logits, dim=-1, index=teacher_topk_indices) - student_logits.div_(temperature) - teacher_logits = teacher_topk_logprobs / temperature - temperature = 1.0 + teacher_logits = teacher_topk_logprobs elif topk is not None and teacher_logits is not None: # Local teacher: select top-k from teacher, gather corresponding student logits teacher_logits = teacher_logits.to(student_logits.device) teacher_logits, topk_idx = torch.topk(teacher_logits, k=topk, dim=-1) - teacher_logits.div_(temperature) student_logits = torch.gather(student_logits, dim=-1, index=topk_idx) - student_logits.div_(temperature) - temperature = 1.0 if teacher_logits is not None and teacher_logits.device != student_logits.device: teacher_logits = teacher_logits.to(student_logits.device) @@ -210,12 +203,16 @@ def _generalized_jsd_loss( num_valid = mask.sum() # ``[mask]`` already created fresh storage, so in-place divide is safe # and avoids an extra [num_valid, V] allocation. - student_logits.div_(temperature) - teacher_logits.div_(temperature) + if temperature != 1.0: + student_logits.div_(temperature) + teacher_logits.div_(temperature) else: # Keep logits, may be an infer scenario - student_logits = student_logits.reshape(-1, student_logits.size(-1)) / temperature - teacher_logits = teacher_logits.reshape(-1, teacher_logits.size(-1)) / temperature + student_logits = student_logits.reshape(-1, student_logits.size(-1)) + teacher_logits = teacher_logits.reshape(-1, teacher_logits.size(-1)) + if temperature != 1.0: + student_logits = student_logits / temperature + teacher_logits = teacher_logits / temperature num_valid = student_logits.size(0) if num_valid == 0: diff --git a/src/twinkle/loss/grpo.py b/src/twinkle/loss/grpo.py index 629dde55d..3b599322e 100644 --- a/src/twinkle/loss/grpo.py +++ b/src/twinkle/loss/grpo.py @@ -1101,7 +1101,7 @@ def _reduce_loss( batch_pos_mask = pos_mask[valid_mask] batch_neg_mask = neg_mask[valid_mask] - scaled_scores = batch_scores / self.real_tau + scaled_scores = batch_scores if self.real_tau == 1.0 else batch_scores / self.real_tau zeros = torch.zeros(batch_scores.size(0), 1, device=batch_scores.device, dtype=batch_scores.dtype) # Negative loss: log(1 + sum(exp(S_neg))) diff --git a/src/twinkle/loss/infonce.py b/src/twinkle/loss/infonce.py index 928a36766..8fc654a51 100644 --- a/src/twinkle/loss/infonce.py +++ b/src/twinkle/loss/infonce.py @@ -294,13 +294,17 @@ def _intra_sample_loss(self, split_tensors, can_batched) -> torch.Tensor: """InfoNCE with only the per-sample negatives (no cross-sample sharing).""" if can_batched: sentences = torch.stack(split_tensors, dim=0) # [B, neg+2, D] - similarity_matrix = torch.matmul(sentences[:, 0:1], sentences[:, 1:].transpose(1, 2)) / self.temperature + similarity_matrix = torch.matmul(sentences[:, 0:1], sentences[:, 1:].transpose(1, 2)) + if self.temperature != 1.0: + similarity_matrix = similarity_matrix / self.temperature labels = torch.zeros(len(split_tensors), dtype=torch.int64, device=sentences.device) return nn.CrossEntropyLoss()(similarity_matrix.squeeze(1), labels) loss = 0 for tensor in split_tensors: - similarity_matrix = torch.matmul(tensor[0], tensor[1:].T) / self.temperature + similarity_matrix = torch.matmul(tensor[0], tensor[1:].T) + if self.temperature != 1.0: + similarity_matrix = similarity_matrix / self.temperature labels = torch.tensor(0, device=tensor.device) loss = loss + nn.CrossEntropyLoss()(similarity_matrix, labels) return loss / len(split_tensors) @@ -353,7 +357,9 @@ def _in_batch_loss_batched(self, split_tensors) -> torch.Tensor: else: similarity_matrix = torch.cat(logits_list, dim=1) - return nn.CrossEntropyLoss()(similarity_matrix / self.temperature, labels) + if self.temperature != 1.0: + similarity_matrix = similarity_matrix / self.temperature + return nn.CrossEntropyLoss()(similarity_matrix, labels) def _in_batch_loss_unbatched(self, split_tensors) -> torch.Tensor: # docs from every sample concatenated as a shared negative bank @@ -383,7 +389,9 @@ def _in_batch_loss_unbatched(self, split_tensors) -> torch.Tensor: dd_vec[length] = float('-inf') logits_parts.append(dd_vec) - logits_row = torch.cat(logits_parts, dim=-1) / self.temperature + logits_row = torch.cat(logits_parts, dim=-1) + if self.temperature != 1.0: + logits_row = logits_row / self.temperature loss = loss + nn.CrossEntropyLoss()(logits_row.unsqueeze(0), target.unsqueeze(0)) length += tensor.size(0) - 1 return loss / len(split_tensors) diff --git a/src/twinkle/loss/reranker.py b/src/twinkle/loss/reranker.py index 533290981..bfa278496 100644 --- a/src/twinkle/loss/reranker.py +++ b/src/twinkle/loss/reranker.py @@ -108,7 +108,9 @@ def __call__(self, inputs, outputs, **kwargs) -> LossOutput: if labels[start] != 1: continue target = torch.zeros(1, dtype=torch.long, device=logits.device) - group_loss = loss_fct((group_logits / self.temperature).unsqueeze(0), target) + if self.temperature != 1.0: + group_logits = group_logits / self.temperature + group_loss = loss_fct(group_logits.unsqueeze(0), target) total_loss = group_loss if total_loss is None else total_loss + group_loss num_groups += 1 diff --git a/src/twinkle/model/megatron/megatron.py b/src/twinkle/model/megatron/megatron.py index 01793197e..19efcdf45 100644 --- a/src/twinkle/model/megatron/megatron.py +++ b/src/twinkle/model/megatron/megatron.py @@ -1045,7 +1045,8 @@ def forward_step_func(data_iterator, model): loss_mask = (labels != -100).bool() masked_labels = labels.clone() masked_labels[~loss_mask] = 0 - output_tensor.div_(temperature) + if temperature != 1.0: + output_tensor.div_(temperature) if _loss_require_entropy: logps, entropies = selective_log_softmax(output_tensor, masked_labels, return_entropy=True) else: diff --git a/src/twinkle/utils/torch_utils.py b/src/twinkle/utils/torch_utils.py index 21334a0f5..9a7b3e8c8 100644 --- a/src/twinkle/utils/torch_utils.py +++ b/src/twinkle/utils/torch_utils.py @@ -412,12 +412,15 @@ def replayed_selective_log_softmax( positions[row_ids, 0], positions[row_ids, 1], kept_token_ids, - ].float() / temperature + ].float() selected_logits = logits[ positions[:, 0], positions[:, 1], sampled_labels, - ].float() / temperature + ].float() + if temperature != 1.0: + kept_logits = kept_logits / temperature + selected_logits = selected_logits / temperature # Use max-shifted log-sum-exp for numerically stable restricted softmax. row_max = torch.full( From 34d8a550a17061478af17f62b0b055f84e7b75f2 Mon Sep 17 00:00:00 2001 From: QiyuanChen <72334646+qychen2001@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:27:04 +0800 Subject: [PATCH 4/4] refactor: limit additional temperature fast path to Megatron Co-authored-by: Codex --- src/twinkle/loss/gkd.py | 21 ++++++++++++--------- src/twinkle/loss/grpo.py | 2 +- src/twinkle/loss/infonce.py | 16 ++++------------ src/twinkle/loss/reranker.py | 4 +--- src/twinkle/utils/torch_utils.py | 7 ++----- 5 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/twinkle/loss/gkd.py b/src/twinkle/loss/gkd.py index 95f16b7f4..49e48366b 100644 --- a/src/twinkle/loss/gkd.py +++ b/src/twinkle/loss/gkd.py @@ -170,17 +170,24 @@ def _generalized_jsd_loss( import torch.nn.functional as F # ── Top-k reduction ────────────────────────────────────────────────── + # Top-k mode: gather/topk first to get small [*, k] tensors, then scale in-place if teacher_topk_logprobs is not None and teacher_topk_indices is not None: # Remote API teacher: teacher already provides top-k log-probs (T=1). + # Gather student logits at teacher's top-k indices, then scale in-place. teacher_topk_indices = teacher_topk_indices.to(student_logits.device) teacher_topk_logprobs = teacher_topk_logprobs.to(student_logits.device) student_logits = torch.gather(student_logits, dim=-1, index=teacher_topk_indices) - teacher_logits = teacher_topk_logprobs + student_logits.div_(temperature) + teacher_logits = teacher_topk_logprobs / temperature + temperature = 1.0 elif topk is not None and teacher_logits is not None: # Local teacher: select top-k from teacher, gather corresponding student logits teacher_logits = teacher_logits.to(student_logits.device) teacher_logits, topk_idx = torch.topk(teacher_logits, k=topk, dim=-1) + teacher_logits.div_(temperature) student_logits = torch.gather(student_logits, dim=-1, index=topk_idx) + student_logits.div_(temperature) + temperature = 1.0 if teacher_logits is not None and teacher_logits.device != student_logits.device: teacher_logits = teacher_logits.to(student_logits.device) @@ -203,16 +210,12 @@ def _generalized_jsd_loss( num_valid = mask.sum() # ``[mask]`` already created fresh storage, so in-place divide is safe # and avoids an extra [num_valid, V] allocation. - if temperature != 1.0: - student_logits.div_(temperature) - teacher_logits.div_(temperature) + student_logits.div_(temperature) + teacher_logits.div_(temperature) else: # Keep logits, may be an infer scenario - student_logits = student_logits.reshape(-1, student_logits.size(-1)) - teacher_logits = teacher_logits.reshape(-1, teacher_logits.size(-1)) - if temperature != 1.0: - student_logits = student_logits / temperature - teacher_logits = teacher_logits / temperature + student_logits = student_logits.reshape(-1, student_logits.size(-1)) / temperature + teacher_logits = teacher_logits.reshape(-1, teacher_logits.size(-1)) / temperature num_valid = student_logits.size(0) if num_valid == 0: diff --git a/src/twinkle/loss/grpo.py b/src/twinkle/loss/grpo.py index 3b599322e..629dde55d 100644 --- a/src/twinkle/loss/grpo.py +++ b/src/twinkle/loss/grpo.py @@ -1101,7 +1101,7 @@ def _reduce_loss( batch_pos_mask = pos_mask[valid_mask] batch_neg_mask = neg_mask[valid_mask] - scaled_scores = batch_scores if self.real_tau == 1.0 else batch_scores / self.real_tau + scaled_scores = batch_scores / self.real_tau zeros = torch.zeros(batch_scores.size(0), 1, device=batch_scores.device, dtype=batch_scores.dtype) # Negative loss: log(1 + sum(exp(S_neg))) diff --git a/src/twinkle/loss/infonce.py b/src/twinkle/loss/infonce.py index 8fc654a51..928a36766 100644 --- a/src/twinkle/loss/infonce.py +++ b/src/twinkle/loss/infonce.py @@ -294,17 +294,13 @@ def _intra_sample_loss(self, split_tensors, can_batched) -> torch.Tensor: """InfoNCE with only the per-sample negatives (no cross-sample sharing).""" if can_batched: sentences = torch.stack(split_tensors, dim=0) # [B, neg+2, D] - similarity_matrix = torch.matmul(sentences[:, 0:1], sentences[:, 1:].transpose(1, 2)) - if self.temperature != 1.0: - similarity_matrix = similarity_matrix / self.temperature + similarity_matrix = torch.matmul(sentences[:, 0:1], sentences[:, 1:].transpose(1, 2)) / self.temperature labels = torch.zeros(len(split_tensors), dtype=torch.int64, device=sentences.device) return nn.CrossEntropyLoss()(similarity_matrix.squeeze(1), labels) loss = 0 for tensor in split_tensors: - similarity_matrix = torch.matmul(tensor[0], tensor[1:].T) - if self.temperature != 1.0: - similarity_matrix = similarity_matrix / self.temperature + similarity_matrix = torch.matmul(tensor[0], tensor[1:].T) / self.temperature labels = torch.tensor(0, device=tensor.device) loss = loss + nn.CrossEntropyLoss()(similarity_matrix, labels) return loss / len(split_tensors) @@ -357,9 +353,7 @@ def _in_batch_loss_batched(self, split_tensors) -> torch.Tensor: else: similarity_matrix = torch.cat(logits_list, dim=1) - if self.temperature != 1.0: - similarity_matrix = similarity_matrix / self.temperature - return nn.CrossEntropyLoss()(similarity_matrix, labels) + return nn.CrossEntropyLoss()(similarity_matrix / self.temperature, labels) def _in_batch_loss_unbatched(self, split_tensors) -> torch.Tensor: # docs from every sample concatenated as a shared negative bank @@ -389,9 +383,7 @@ def _in_batch_loss_unbatched(self, split_tensors) -> torch.Tensor: dd_vec[length] = float('-inf') logits_parts.append(dd_vec) - logits_row = torch.cat(logits_parts, dim=-1) - if self.temperature != 1.0: - logits_row = logits_row / self.temperature + logits_row = torch.cat(logits_parts, dim=-1) / self.temperature loss = loss + nn.CrossEntropyLoss()(logits_row.unsqueeze(0), target.unsqueeze(0)) length += tensor.size(0) - 1 return loss / len(split_tensors) diff --git a/src/twinkle/loss/reranker.py b/src/twinkle/loss/reranker.py index bfa278496..533290981 100644 --- a/src/twinkle/loss/reranker.py +++ b/src/twinkle/loss/reranker.py @@ -108,9 +108,7 @@ def __call__(self, inputs, outputs, **kwargs) -> LossOutput: if labels[start] != 1: continue target = torch.zeros(1, dtype=torch.long, device=logits.device) - if self.temperature != 1.0: - group_logits = group_logits / self.temperature - group_loss = loss_fct(group_logits.unsqueeze(0), target) + group_loss = loss_fct((group_logits / self.temperature).unsqueeze(0), target) total_loss = group_loss if total_loss is None else total_loss + group_loss num_groups += 1 diff --git a/src/twinkle/utils/torch_utils.py b/src/twinkle/utils/torch_utils.py index 9a7b3e8c8..21334a0f5 100644 --- a/src/twinkle/utils/torch_utils.py +++ b/src/twinkle/utils/torch_utils.py @@ -412,15 +412,12 @@ def replayed_selective_log_softmax( positions[row_ids, 0], positions[row_ids, 1], kept_token_ids, - ].float() + ].float() / temperature selected_logits = logits[ positions[:, 0], positions[:, 1], sampled_labels, - ].float() - if temperature != 1.0: - kept_logits = kept_logits / temperature - selected_logits = selected_logits / temperature + ].float() / temperature # Use max-shifted log-sum-exp for numerically stable restricted softmax. row_max = torch.full(