Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions src/twinkle/checkpoint_engine/nccl_checkpoint_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,10 @@ def _run(self):
# A stale wire-probe from the subscription rendezvous: drain it and keep waiting for
# the manifest topic (the slow-joiner re-probe leaves extra probes in a fast receiver).

# Broadcast tensor data via NCCL
_pg_broadcast(self.pg, self.bucket, src=0)
# An empty final bucket still sends its manifest to terminate the stream.
length = self.metadata['length']
if length:
_pg_broadcast(self.pg, self.bucket[:length], src=0)

async def wait_for_complete(self) -> dict[str, TensorMeta]:
"""Wait for the broadcast operation to complete.
Expand Down Expand Up @@ -457,7 +459,8 @@ async def send_weights(
bucket=send_buf,
metadata={
'bucket_meta': bucket_meta,
'is_last': False
'is_last': False,
'length': offset,
},
socket=self.socket,
topic=self.topic,
Expand Down Expand Up @@ -496,7 +499,8 @@ async def send_weights(
bucket=send_buf,
metadata={
'bucket_meta': bucket_meta,
'is_last': True
'is_last': True,
'length': offset,
},
socket=self.socket,
topic=self.topic,
Expand Down Expand Up @@ -536,7 +540,7 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No
auth_key=self.auth_key,
)
metadata = await broadcast_op.wait_for_complete()
total_bytes += self.bucket_size
total_bytes += metadata['length']
total_params += len(metadata['bucket_meta'])

# Swap buffers
Expand All @@ -563,7 +567,7 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No

# 3. Wait for next bucket
metadata = await broadcast_op.wait_for_complete()
total_bytes += self.bucket_size
total_bytes += metadata['length']
total_params += len(metadata['bucket_meta'])

# 4. Swap buffers
Expand All @@ -580,5 +584,5 @@ async def receive_weights(self, ) -> AsyncGenerator[tuple[str, torch.Tensor], No
elapsed = time.time() - start_time
bandwidth = total_bytes / elapsed / (1024 * 1024 * 1024)
logger.info(f'receive_weights done: rank={self.rank}, '
f'params={total_params}, '
f'params={total_params}, bytes={total_bytes}, '
f'time={elapsed:.2f}s, bandwidth={bandwidth:.2f} GB/s')
3 changes: 2 additions & 1 deletion src/twinkle/model/megatron/megatron.py
Original file line number Diff line number Diff line change
Expand Up @@ -1045,7 +1045,8 @@ def forward_step_func(data_iterator, model):
loss_mask = (labels != -100).bool()
masked_labels = labels.clone()
masked_labels[~loss_mask] = 0
output_tensor.div_(temperature)
if temperature != 1.0:
output_tensor.div_(temperature)
if _loss_require_entropy:
logps, entropies = selective_log_softmax(output_tensor, masked_labels, return_entropy=True)
else:
Expand Down
3 changes: 2 additions & 1 deletion src/twinkle/model/transformers/transformers.py
Original file line number Diff line number Diff line change
Expand Up @@ -911,7 +911,8 @@ def _postprocess_forward_outputs(self, outputs, *, labels, task, temperature, sa
loss_mask = (labels != -100).bool()
masked_labels = labels.masked_fill(~loss_mask, 0)
logits = outputs['logits']
logits.div_(temperature)
if temperature != 1.0:
logits.div_(temperature)
if loss_require_entropy:
outputs['logps'], outputs['entropies'] = selective_log_softmax(
logits, masked_labels, return_entropy=True)
Expand Down
53 changes: 53 additions & 0 deletions tests/checkpoint_engine/test_nccl_bucket_payload.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
# Copyright (c) ModelScope Contributors. All rights reserved.
"""Exercise the full-sync bucket protocol with CPU buffers."""

import asyncio
from collections import deque
from types import SimpleNamespace

import pytest
import torch

from twinkle.checkpoint_engine import nccl_checkpoint_engine as nccl


@pytest.mark.parametrize('sizes, broadcast_sizes', [([], []), ([0], []), ([4], [16]), ([3, 2, 3], [12, 8, 12])])
def test_nccl_broadcast_uses_payload_length(monkeypatch, sizes, broadcast_sizes):
manifests, payloads = deque(), deque()
sent_sizes, received_sizes = [], []
monkeypatch.setattr(torch.cuda, 'synchronize', lambda: None)
monkeypatch.setattr(nccl, 'send_auth_pyobj', lambda socket, metadata, *args, **kwargs: manifests.append(metadata))
monkeypatch.setattr(nccl, 'recv_auth_pyobj', lambda *args, **kwargs: manifests.popleft())

def broadcast(rank, bucket, src):
if rank == 0:
sent_sizes.append(bucket.numel())
payloads.append(bucket.clone())
else:
received_sizes.append(bucket.numel())
payload = payloads.popleft()
assert payload.shape == bucket.shape
bucket.copy_(payload)

monkeypatch.setattr(nccl, '_pg_broadcast', broadcast)

def engine(rank):
engine = nccl.NCCLCheckpointEngine(bucket_size=16)
engine.rank = engine._pg = rank
engine.send_buf = torch.empty(16, dtype=torch.uint8)
engine.recv_buf = torch.empty_like(engine.send_buf)
engine.socket = SimpleNamespace(send_string=lambda *args, **kwargs: None, recv_string=lambda: engine.topic)
return engine

weights = {str(i): torch.arange(n, dtype=torch.float32) + i for i, n in enumerate(sizes)}

async def roundtrip():
await engine(0).send_weights(iter(weights.items()))
return {k: v.clone() async for k, v in engine(1).receive_weights()}

received = asyncio.run(roundtrip())
assert sent_sizes == received_sizes == broadcast_sizes
assert received.keys() == weights.keys()
for name in weights:
torch.testing.assert_close(received[name], weights[name], rtol=0, atol=0)
assert not manifests and not payloads
Loading