From bdc34f825e6ab375fa6fc3365f418b32792c52cc Mon Sep 17 00:00:00 2001 From: dcx <65224225+diaochengxi@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:03:33 +0000 Subject: [PATCH] fix(kernel): align fused entropy backward with native rounding --- .../functional.py | 14 +++--- .../kernels.py | 43 ++++++++----------- .../fused_linear_selective_log_softmax/npu.py | 37 +++++++++++----- ...test_fused_linear_selective_log_softmax.py | 13 +++++- 4 files changed, 62 insertions(+), 45 deletions(-) diff --git a/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/functional.py b/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/functional.py index 9ce106b7b..9c85ad442 100644 --- a/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/functional.py +++ b/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/functional.py @@ -56,13 +56,13 @@ def fused_linear_selective_log_softmax_reference(x, weight, targets, temperature def fused_linear_selective_log_softmax(x, weight, targets, temperature=1.0, return_entropy=False): """Compute selected log-probabilities from hidden states without exposing logits. - The Ascend Triton implementation owns the logits workspace and reuses it for its - gradient. It supports one first-order backward per forward. FP32 hidden states - with a BF16 weight follow the usual mixed-precision cast. FP32 inputs and - non-NPU devices use the native PyTorch reference: the strict FP32 gradient - tolerance requires native-order reductions, which measured slower than the - native pipeline itself, so only the BF16 path ships a fused kernel. The BF16 - NPU path also falls back to the reference when the optional Triton runtime is + The Ascend Triton implementation owns the BF16 logits workspace and reuses it + for log-probability gradients. Entropy backward follows PyTorch's FP32 softmax + and BF16 division rounding so its gradients match the reference under the + strict BF16 tolerance. It supports one first-order backward per forward. FP32 + hidden states with a BF16 weight follow the usual mixed-precision cast. FP32 + inputs and non-NPU devices use the native PyTorch reference. The BF16 NPU path + also falls back to the reference when the optional Triton runtime is unavailable, rather than failing after the model forward. """ temperature = float(temperature) diff --git a/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/kernels.py b/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/kernels.py index 319f20a7e..a2d7c5a7b 100644 --- a/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/kernels.py +++ b/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/kernels.py @@ -10,7 +10,7 @@ @triton.jit -def stats_row_kernel(LOGITS, TARGETS, LSE, LOGPS, V, BLOCK_SIZE: tl.constexpr): +def stats_row_kernel(LOGITS, TARGETS, ROW_MAX, ROW_SUM, LSE, LOGPS, V, BLOCK_SIZE: tl.constexpr): row = tl.program_id(0).to(tl.int64) base = row * V target = tl.load(TARGETS + row) @@ -28,66 +28,57 @@ def stats_row_kernel(LOGITS, TARGETS, LSE, LOGPS, V, BLOCK_SIZE: tl.constexpr): tl.where((tl.arange(0, BLOCK_SIZE) == local_target) & (local_target >= 0) & (local_target < BLOCK_SIZE), values, 0.0), 0) lse = running_max + tl.log(running_sum) + tl.store(ROW_MAX + row, running_max) + tl.store(ROW_SUM + row, running_sum) tl.store(LSE + row, lse) tl.store(LOGPS + row, target_logit - lse) @triton.jit -def backward_row_kernel(LOGITS, TARGETS, LSE, EXPECTED, GRAD_LOGPS, GRAD_ENTROPIES, V, HAS_ENTROPY: tl.constexpr, - BLOCK_SIZE: tl.constexpr): +def backward_row_kernel(LOGITS, TARGETS, ROW_MAX, ROW_SUM, GRAD_LOGPS, V, BLOCK_SIZE: tl.constexpr): row = tl.program_id(0).to(tl.int64) base = row * V target = tl.load(TARGETS + row) - lse = tl.load(LSE + row) + row_max = tl.load(ROW_MAX + row) + row_sum = tl.load(ROW_SUM + row) grad_logp = tl.load(GRAD_LOGPS + row).to(tl.float32) - if HAS_ENTROPY: - grad_entropy = tl.load(GRAD_ENTROPIES + row).to(tl.float32) - expected = tl.load(EXPECTED + row) - else: - grad_entropy = 0.0 - expected = 0.0 for i in range(0, tl.cdiv(V, BLOCK_SIZE)): offsets = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) mask = offsets < V logits = tl.load(LOGITS + base + offsets, mask=mask, other=-float('inf')).to(tl.float32) - probabilities = tl.exp(logits - lse) + probabilities = tl.exp(logits - row_max) / row_sum grad = -(probabilities * grad_logp) - if HAS_ENTROPY: - grad += probabilities * (grad_entropy * (expected - logits)) grad = tl.where(offsets == target, grad + grad_logp, grad) tl.store(LOGITS + base + offsets, grad.to(LOGITS.dtype.element_ty), mask=mask) @triton.jit -def expected_row_kernel(LOGITS, LSE, EXPECTED, V, BLOCK_SIZE: tl.constexpr): +def expected_row_kernel(LOGITS, ROW_MAX, ROW_SUM, EXPECTED, V, BLOCK_SIZE: tl.constexpr): row = tl.program_id(0).to(tl.int64) base = row * V - lse = tl.load(LSE + row) + row_max = tl.load(ROW_MAX + row) + row_sum = tl.load(ROW_SUM + row) running = 0.0 for i in range(0, tl.cdiv(V, BLOCK_SIZE)): offsets = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) valid = offsets < V logits = tl.load(LOGITS + base + offsets, mask=valid, other=0.0).to(tl.float32) # Masked lanes contribute 0 to both factors, avoiding exp(-inf) * (-inf) = NaN. - probabilities = tl.where(valid, tl.exp(logits - lse), 0.0) + probabilities = tl.where(valid, tl.exp(logits - row_max) / row_sum, 0.0) running += tl.sum(probabilities * tl.where(valid, logits, 0.0), 0) tl.store(EXPECTED + row, running) -def expected_launch(logits, lse, expected): +def expected_launch(logits, row_max, row_sum, expected): rows, vocab = logits.shape - expected_row_kernel[(rows, )](logits, lse, expected, vocab, BLOCK, num_warps=NUM_WARPS) + expected_row_kernel[(rows, )](logits, row_max, row_sum, expected, vocab, BLOCK, num_warps=NUM_WARPS) -def stats_launch(logits, targets, lse, logps): +def stats_launch(logits, targets, row_max, row_sum, lse, logps): rows, vocab = logits.shape - stats_row_kernel[(rows, )](logits, targets, lse, logps, vocab, BLOCK, num_warps=NUM_WARPS) + stats_row_kernel[(rows, )](logits, targets, row_max, row_sum, lse, logps, vocab, BLOCK, num_warps=NUM_WARPS) -def backward_launch(logits, targets, lse, expected, grad_logps, grad_entropies): +def backward_launch(logits, targets, row_max, row_sum, grad_logps): rows, vocab = logits.shape - has_entropy = grad_entropies is not None - if not has_entropy: - grad_entropies = grad_logps - backward_row_kernel[(rows, )]( - logits, targets, lse, expected, grad_logps, grad_entropies, vocab, has_entropy, BLOCK, num_warps=NUM_WARPS) + backward_row_kernel[(rows, )](logits, targets, row_max, row_sum, grad_logps, vocab, BLOCK, num_warps=NUM_WARPS) diff --git a/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/npu.py b/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/npu.py index 5912ec952..760312f75 100644 --- a/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/npu.py +++ b/src/twinkle/kernel/ops/fused_linear_selective_log_softmax/npu.py @@ -14,17 +14,19 @@ def forward(ctx, x, weight, targets, temperature, return_entropy): logits = torch.mm(x, weight.t()) logits.div_(temperature) rows = x.shape[0] + row_max = torch.empty(rows, dtype=torch.float32, device=x.device) + row_sum = torch.empty(rows, dtype=torch.float32, device=x.device) lse = torch.empty(rows, dtype=torch.float32, device=x.device) logps = torch.empty(rows, dtype=torch.float32, device=x.device) expected = torch.empty_like(logps) if return_entropy else lse - stats_launch(logits, targets, lse, logps) + stats_launch(logits, targets, row_max, row_sum, lse, logps) if return_entropy: - expected_launch(logits, lse, expected) + expected_launch(logits, row_max, row_sum, expected) entropies = lse - expected else: entropies = None - ctx.save_for_backward(logits, targets, lse, expected, x, weight) + ctx.save_for_backward(logits, targets, row_max, row_sum, x, weight) ctx.return_entropy = return_entropy ctx.temperature = temperature ctx.consumed = False @@ -37,21 +39,36 @@ def backward(ctx, grad_logps, grad_entropies=None): if ctx.consumed: raise RuntimeError('fused linear selective log-softmax does not support repeated backward') ctx.consumed = True - logits, targets, lse, expected, x, weight = ctx.saved_tensors + logits, targets, row_max, row_sum, x, weight = ctx.saved_tensors if grad_logps is None: - grad_logps = torch.zeros_like(lse) + grad_logps = torch.zeros_like(row_max) if ctx.return_entropy and grad_entropies is None: - grad_entropies = torch.zeros_like(lse) + grad_entropies = torch.zeros_like(row_max) if not ctx.return_entropy: grad_entropies = None if grad_entropies is not None: grad_entropies = grad_entropies.contiguous() grad_logps = grad_logps.contiguous() - backward_launch(logits, targets, lse, expected, grad_logps, grad_entropies) - logits.div_(ctx.temperature) - grad_x = torch.mm(logits, weight) if ctx.needs_input_grad[0] else None - grad_weight = torch.mm(logits.t(), x) if ctx.needs_input_grad[1] else None + if ctx.return_entropy: + # Match the reference graph's FP32 softmax and BF16 division rounding. + scaled_logits = logits.float() + probabilities = torch.softmax(scaled_logits, dim=-1) + expected_logits = (probabilities * scaled_logits).sum(dim=-1, keepdim=True) + grad_logits = scaled_logits.neg().add_(expected_logits) + grad_logits.mul_(grad_entropies.unsqueeze(-1)) + grad_logits.sub_(grad_logps.unsqueeze(-1)) + grad_logits.mul_(probabilities) + grad_logits.scatter_add_(-1, targets.unsqueeze(-1), grad_logps.unsqueeze(-1)) + grad_logits = grad_logits.bfloat16() + grad_logits.div_(ctx.temperature) + grad_x = torch.mm(grad_logits, weight) if ctx.needs_input_grad[0] else None + grad_weight = torch.mm(grad_logits.t(), x) if ctx.needs_input_grad[1] else None + else: + backward_launch(logits, targets, row_max, row_sum, grad_logps) + logits.div_(ctx.temperature) + grad_x = torch.mm(logits, weight) if ctx.needs_input_grad[0] else None + grad_weight = torch.mm(logits.t(), x) if ctx.needs_input_grad[1] else None return grad_x, grad_weight, None, None, None diff --git a/tests/kernel/ops/test_fused_linear_selective_log_softmax.py b/tests/kernel/ops/test_fused_linear_selective_log_softmax.py index 311d7e17d..6b3602d0b 100644 --- a/tests/kernel/ops/test_fused_linear_selective_log_softmax.py +++ b/tests/kernel/ops/test_fused_linear_selective_log_softmax.py @@ -29,9 +29,11 @@ def _native(x, weight, targets, temperature, return_entropy): return logps, entropies -def _check_forward_backward(device, dtype, return_entropy, shape=(17, 12, 5000)): +def _check_forward_backward(device, dtype, return_entropy, shape=(17, 12, 5000), seed=2026): rows, hidden, vocab = shape - torch.manual_seed(2026) + torch.manual_seed(seed) + if device == 'npu': + torch.npu.manual_seed(seed) x = torch.randn(rows, hidden, device=device, dtype=dtype, requires_grad=True) weight = torch.randn(vocab, hidden, device=device, dtype=dtype, requires_grad=True) targets = torch.randint(0, vocab, (rows, ), device=device) @@ -161,6 +163,13 @@ def test_npu_real_vocab_matches_native(dtype, return_entropy): _check_forward_backward('npu', dtype, return_entropy, shape=(128, 2048, 151936)) +@pytest.mark.skipif(not _NPU_OK, reason='torch_npu unavailable') +def test_npu_real_vocab_entropy_backward_normalizes_probabilities(): + # Regression coverage for the large-logit entropy path. Reusing the forward + # row sum avoids an unnormalized exp(logits - lse) approximation in backward. + _check_forward_backward('npu', torch.bfloat16, True, shape=(256, 2048, 151936), seed=20261117) + + def test_float_targets_are_rejected(): x = torch.randn(4, 8) weight = torch.randn(31, 8)