Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,13 @@ def fused_linear_selective_log_softmax_reference(x, weight, targets, temperature
def fused_linear_selective_log_softmax(x, weight, targets, temperature=1.0, return_entropy=False):
"""Compute selected log-probabilities from hidden states without exposing logits.

The Ascend Triton implementation owns the logits workspace and reuses it for its
gradient. It supports one first-order backward per forward. FP32 hidden states
with a BF16 weight follow the usual mixed-precision cast. FP32 inputs and
non-NPU devices use the native PyTorch reference: the strict FP32 gradient
tolerance requires native-order reductions, which measured slower than the
native pipeline itself, so only the BF16 path ships a fused kernel. The BF16
NPU path also falls back to the reference when the optional Triton runtime is
The Ascend Triton implementation owns the BF16 logits workspace and reuses it
for log-probability gradients. Entropy backward follows PyTorch's FP32 softmax
and BF16 division rounding so its gradients match the reference under the
strict BF16 tolerance. It supports one first-order backward per forward. FP32
hidden states with a BF16 weight follow the usual mixed-precision cast. FP32
inputs and non-NPU devices use the native PyTorch reference. The BF16 NPU path
also falls back to the reference when the optional Triton runtime is
unavailable, rather than failing after the model forward.
"""
temperature = float(temperature)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@


@triton.jit
def stats_row_kernel(LOGITS, TARGETS, LSE, LOGPS, V, BLOCK_SIZE: tl.constexpr):
def stats_row_kernel(LOGITS, TARGETS, ROW_MAX, ROW_SUM, LSE, LOGPS, V, BLOCK_SIZE: tl.constexpr):
row = tl.program_id(0).to(tl.int64)
base = row * V
target = tl.load(TARGETS + row)
Expand All @@ -28,66 +28,57 @@ def stats_row_kernel(LOGITS, TARGETS, LSE, LOGPS, V, BLOCK_SIZE: tl.constexpr):
tl.where((tl.arange(0, BLOCK_SIZE) == local_target) & (local_target >= 0) & (local_target < BLOCK_SIZE),
values, 0.0), 0)
lse = running_max + tl.log(running_sum)
tl.store(ROW_MAX + row, running_max)
tl.store(ROW_SUM + row, running_sum)
tl.store(LSE + row, lse)
tl.store(LOGPS + row, target_logit - lse)


@triton.jit
def backward_row_kernel(LOGITS, TARGETS, LSE, EXPECTED, GRAD_LOGPS, GRAD_ENTROPIES, V, HAS_ENTROPY: tl.constexpr,
BLOCK_SIZE: tl.constexpr):
def backward_row_kernel(LOGITS, TARGETS, ROW_MAX, ROW_SUM, GRAD_LOGPS, V, BLOCK_SIZE: tl.constexpr):
row = tl.program_id(0).to(tl.int64)
base = row * V
target = tl.load(TARGETS + row)
lse = tl.load(LSE + row)
row_max = tl.load(ROW_MAX + row)
row_sum = tl.load(ROW_SUM + row)
grad_logp = tl.load(GRAD_LOGPS + row).to(tl.float32)
if HAS_ENTROPY:
grad_entropy = tl.load(GRAD_ENTROPIES + row).to(tl.float32)
expected = tl.load(EXPECTED + row)
else:
grad_entropy = 0.0
expected = 0.0
for i in range(0, tl.cdiv(V, BLOCK_SIZE)):
offsets = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < V
logits = tl.load(LOGITS + base + offsets, mask=mask, other=-float('inf')).to(tl.float32)
probabilities = tl.exp(logits - lse)
probabilities = tl.exp(logits - row_max) / row_sum
grad = -(probabilities * grad_logp)
if HAS_ENTROPY:
grad += probabilities * (grad_entropy * (expected - logits))
grad = tl.where(offsets == target, grad + grad_logp, grad)
tl.store(LOGITS + base + offsets, grad.to(LOGITS.dtype.element_ty), mask=mask)


@triton.jit
def expected_row_kernel(LOGITS, LSE, EXPECTED, V, BLOCK_SIZE: tl.constexpr):
def expected_row_kernel(LOGITS, ROW_MAX, ROW_SUM, EXPECTED, V, BLOCK_SIZE: tl.constexpr):
row = tl.program_id(0).to(tl.int64)
base = row * V
lse = tl.load(LSE + row)
row_max = tl.load(ROW_MAX + row)
row_sum = tl.load(ROW_SUM + row)
running = 0.0
for i in range(0, tl.cdiv(V, BLOCK_SIZE)):
offsets = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
valid = offsets < V
logits = tl.load(LOGITS + base + offsets, mask=valid, other=0.0).to(tl.float32)
# Masked lanes contribute 0 to both factors, avoiding exp(-inf) * (-inf) = NaN.
probabilities = tl.where(valid, tl.exp(logits - lse), 0.0)
probabilities = tl.where(valid, tl.exp(logits - row_max) / row_sum, 0.0)
running += tl.sum(probabilities * tl.where(valid, logits, 0.0), 0)
tl.store(EXPECTED + row, running)


def expected_launch(logits, lse, expected):
def expected_launch(logits, row_max, row_sum, expected):
rows, vocab = logits.shape
expected_row_kernel[(rows, )](logits, lse, expected, vocab, BLOCK, num_warps=NUM_WARPS)
expected_row_kernel[(rows, )](logits, row_max, row_sum, expected, vocab, BLOCK, num_warps=NUM_WARPS)


def stats_launch(logits, targets, lse, logps):
def stats_launch(logits, targets, row_max, row_sum, lse, logps):
rows, vocab = logits.shape
stats_row_kernel[(rows, )](logits, targets, lse, logps, vocab, BLOCK, num_warps=NUM_WARPS)
stats_row_kernel[(rows, )](logits, targets, row_max, row_sum, lse, logps, vocab, BLOCK, num_warps=NUM_WARPS)


def backward_launch(logits, targets, lse, expected, grad_logps, grad_entropies):
def backward_launch(logits, targets, row_max, row_sum, grad_logps):
rows, vocab = logits.shape
has_entropy = grad_entropies is not None
if not has_entropy:
grad_entropies = grad_logps
backward_row_kernel[(rows, )](
logits, targets, lse, expected, grad_logps, grad_entropies, vocab, has_entropy, BLOCK, num_warps=NUM_WARPS)
backward_row_kernel[(rows, )](logits, targets, row_max, row_sum, grad_logps, vocab, BLOCK, num_warps=NUM_WARPS)
37 changes: 27 additions & 10 deletions src/twinkle/kernel/ops/fused_linear_selective_log_softmax/npu.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,19 @@ def forward(ctx, x, weight, targets, temperature, return_entropy):
logits = torch.mm(x, weight.t())
logits.div_(temperature)
rows = x.shape[0]
row_max = torch.empty(rows, dtype=torch.float32, device=x.device)
row_sum = torch.empty(rows, dtype=torch.float32, device=x.device)
lse = torch.empty(rows, dtype=torch.float32, device=x.device)
logps = torch.empty(rows, dtype=torch.float32, device=x.device)
expected = torch.empty_like(logps) if return_entropy else lse
stats_launch(logits, targets, lse, logps)
stats_launch(logits, targets, row_max, row_sum, lse, logps)
if return_entropy:
expected_launch(logits, lse, expected)
expected_launch(logits, row_max, row_sum, expected)
entropies = lse - expected
else:
entropies = None

ctx.save_for_backward(logits, targets, lse, expected, x, weight)
ctx.save_for_backward(logits, targets, row_max, row_sum, x, weight)
ctx.return_entropy = return_entropy
ctx.temperature = temperature
ctx.consumed = False
Expand All @@ -37,21 +39,36 @@ def backward(ctx, grad_logps, grad_entropies=None):
if ctx.consumed:
raise RuntimeError('fused linear selective log-softmax does not support repeated backward')
ctx.consumed = True
logits, targets, lse, expected, x, weight = ctx.saved_tensors
logits, targets, row_max, row_sum, x, weight = ctx.saved_tensors
if grad_logps is None:
grad_logps = torch.zeros_like(lse)
grad_logps = torch.zeros_like(row_max)
if ctx.return_entropy and grad_entropies is None:
grad_entropies = torch.zeros_like(lse)
grad_entropies = torch.zeros_like(row_max)
if not ctx.return_entropy:
grad_entropies = None

if grad_entropies is not None:
grad_entropies = grad_entropies.contiguous()
grad_logps = grad_logps.contiguous()
backward_launch(logits, targets, lse, expected, grad_logps, grad_entropies)
logits.div_(ctx.temperature)
grad_x = torch.mm(logits, weight) if ctx.needs_input_grad[0] else None
grad_weight = torch.mm(logits.t(), x) if ctx.needs_input_grad[1] else None
if ctx.return_entropy:
# Match the reference graph's FP32 softmax and BF16 division rounding.
scaled_logits = logits.float()
probabilities = torch.softmax(scaled_logits, dim=-1)
expected_logits = (probabilities * scaled_logits).sum(dim=-1, keepdim=True)
grad_logits = scaled_logits.neg().add_(expected_logits)
grad_logits.mul_(grad_entropies.unsqueeze(-1))
grad_logits.sub_(grad_logps.unsqueeze(-1))
grad_logits.mul_(probabilities)
grad_logits.scatter_add_(-1, targets.unsqueeze(-1), grad_logps.unsqueeze(-1))
grad_logits = grad_logits.bfloat16()
grad_logits.div_(ctx.temperature)
grad_x = torch.mm(grad_logits, weight) if ctx.needs_input_grad[0] else None
grad_weight = torch.mm(grad_logits.t(), x) if ctx.needs_input_grad[1] else None
else:
backward_launch(logits, targets, row_max, row_sum, grad_logps)
logits.div_(ctx.temperature)
grad_x = torch.mm(logits, weight) if ctx.needs_input_grad[0] else None
grad_weight = torch.mm(logits.t(), x) if ctx.needs_input_grad[1] else None
return grad_x, grad_weight, None, None, None


Expand Down
13 changes: 11 additions & 2 deletions tests/kernel/ops/test_fused_linear_selective_log_softmax.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@ def _native(x, weight, targets, temperature, return_entropy):
return logps, entropies


def _check_forward_backward(device, dtype, return_entropy, shape=(17, 12, 5000)):
def _check_forward_backward(device, dtype, return_entropy, shape=(17, 12, 5000), seed=2026):
rows, hidden, vocab = shape
torch.manual_seed(2026)
torch.manual_seed(seed)
if device == 'npu':
torch.npu.manual_seed(seed)
x = torch.randn(rows, hidden, device=device, dtype=dtype, requires_grad=True)
weight = torch.randn(vocab, hidden, device=device, dtype=dtype, requires_grad=True)
targets = torch.randint(0, vocab, (rows, ), device=device)
Expand Down Expand Up @@ -161,6 +163,13 @@ def test_npu_real_vocab_matches_native(dtype, return_entropy):
_check_forward_backward('npu', dtype, return_entropy, shape=(128, 2048, 151936))


@pytest.mark.skipif(not _NPU_OK, reason='torch_npu unavailable')
def test_npu_real_vocab_entropy_backward_normalizes_probabilities():
# Regression coverage for the large-logit entropy path. Reusing the forward
# row sum avoids an unnormalized exp(logits - lse) approximation in backward.
_check_forward_backward('npu', torch.bfloat16, True, shape=(256, 2048, 151936), seed=20261117)


def test_float_targets_are_rejected():
x = torch.randn(4, 8)
weight = torch.randn(31, 8)
Expand Down
Loading