diff --git a/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py b/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py index 3490f632b..8a0b9c630 100644 --- a/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py +++ b/src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py @@ -104,8 +104,10 @@ def _run(self): # A stale wire-probe from the subscription rendezvous: drain it and keep waiting for # the manifest topic (the slow-joiner re-probe leaves extra probes in a fast receiver). - # Broadcast tensor data via NCCL - _pg_broadcast(self.pg, self.bucket, src=0) + # An empty final bucket still sends its manifest to terminate the stream. + length = self.metadata['length'] + if length: + _pg_broadcast(self.pg, self.bucket[:length], src=0) async def wait_for_complete(self) -> dict[str, TensorMeta]: """Wait for the broadcast operation to complete. @@ -457,7 +459,8 @@ async def send_weights( bucket=send_buf, metadata={ 'bucket_meta': bucket_meta, - 'is_last': False + 'is_last': False, + 'length': offset, }, socket=self.socket, topic=self.topic, @@ -496,7 +499,8 @@ async def send_weights( bucket=send_buf, metadata={ 'bucket_meta': bucket_meta, - 'is_last': True + 'is_last': True, + 'length': offset, }, socket=self.socket, topic=self.topic, @@ -536,7 +540,7 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No auth_key=self.auth_key, ) metadata = await broadcast_op.wait_for_complete() - total_bytes += self.bucket_size + total_bytes += metadata['length'] total_params += len(metadata['bucket_meta']) # Swap buffers @@ -563,7 +567,7 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No # 3. Wait for next bucket metadata = await broadcast_op.wait_for_complete() - total_bytes += self.bucket_size + total_bytes += metadata['length'] total_params += len(metadata['bucket_meta']) # 4. Swap buffers @@ -580,5 +584,5 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No elapsed = time.time() - start_time bandwidth = total_bytes / elapsed / (1024 * 1024 * 1024) logger.info(f'receive_weights done: rank={self.rank}, ' - f'params={total_params}, ' + f'params={total_params}, bytes={total_bytes}, ' f'time={elapsed:.2f}s, bandwidth={bandwidth:.2f} GB/s') diff --git a/src/twinkle/model/megatron/megatron.py b/src/twinkle/model/megatron/megatron.py index 01793197e..19efcdf45 100644 --- a/src/twinkle/model/megatron/megatron.py +++ b/src/twinkle/model/megatron/megatron.py @@ -1045,7 +1045,8 @@ def forward_step_func(data_iterator, model): loss_mask = (labels != -100).bool() masked_labels = labels.clone() masked_labels[~loss_mask] = 0 - output_tensor.div_(temperature) + if temperature != 1.0: + output_tensor.div_(temperature) if _loss_require_entropy: logps, entropies = selective_log_softmax(output_tensor, masked_labels, return_entropy=True) else: diff --git a/src/twinkle/model/transformers/transformers.py b/src/twinkle/model/transformers/transformers.py index a147f4da2..cb0dedcc9 100644 --- a/src/twinkle/model/transformers/transformers.py +++ b/src/twinkle/model/transformers/transformers.py @@ -911,7 +911,8 @@ def _postprocess_forward_outputs(self, outputs, *, labels, task, temperature, sa loss_mask = (labels != -100).bool() masked_labels = labels.masked_fill(~loss_mask, 0) logits = outputs['logits'] - logits.div_(temperature) + if temperature != 1.0: + logits.div_(temperature) if loss_require_entropy: outputs['logps'], outputs['entropies'] = selective_log_softmax( logits, masked_labels, return_entropy=True) diff --git a/tests/checkpoint_engine/test_nccl_bucket_payload.py b/tests/checkpoint_engine/test_nccl_bucket_payload.py new file mode 100644 index 000000000..695d8cc4d --- /dev/null +++ b/tests/checkpoint_engine/test_nccl_bucket_payload.py @@ -0,0 +1,53 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Exercise the full-sync bucket protocol with CPU buffers.""" + +import asyncio +from collections import deque +from types import SimpleNamespace + +import pytest +import torch + +from twinkle.checkpoint_engine import nccl_checkpoint_engine as nccl + + +@pytest.mark.parametrize('sizes, broadcast_sizes', [([], []), ([0], []), ([4], [16]), ([3, 2, 3], [12, 8, 12])]) +def test_nccl_broadcast_uses_payload_length(monkeypatch, sizes, broadcast_sizes): + manifests, payloads = deque(), deque() + sent_sizes, received_sizes = [], [] + monkeypatch.setattr(torch.cuda, 'synchronize', lambda: None) + monkeypatch.setattr(nccl, 'send_auth_pyobj', lambda socket, metadata, *args, **kwargs: manifests.append(metadata)) + monkeypatch.setattr(nccl, 'recv_auth_pyobj', lambda *args, **kwargs: manifests.popleft()) + + def broadcast(rank, bucket, src): + if rank == 0: + sent_sizes.append(bucket.numel()) + payloads.append(bucket.clone()) + else: + received_sizes.append(bucket.numel()) + payload = payloads.popleft() + assert payload.shape == bucket.shape + bucket.copy_(payload) + + monkeypatch.setattr(nccl, '_pg_broadcast', broadcast) + + def engine(rank): + engine = nccl.NCCLCheckpointEngine(bucket_size=16) + engine.rank = engine._pg = rank + engine.send_buf = torch.empty(16, dtype=torch.uint8) + engine.recv_buf = torch.empty_like(engine.send_buf) + engine.socket = SimpleNamespace(send_string=lambda *args, **kwargs: None, recv_string=lambda: engine.topic) + return engine + + weights = {str(i): torch.arange(n, dtype=torch.float32) + i for i, n in enumerate(sizes)} + + async def roundtrip(): + await engine(0).send_weights(iter(weights.items())) + return {k: v.clone() async for k, v in engine(1).receive_weights()} + + received = asyncio.run(roundtrip()) + assert sent_sizes == received_sizes == broadcast_sizes + assert received.keys() == weights.keys() + for name in weights: + torch.testing.assert_close(received[name], weights[name], rtol=0, atol=0) + assert not manifests and not payloads