Kernels:

kernels-community
/

nvfp4-gemm

Trusted publisher

Kernel card Files Files and versions

xet

Community

kernels-bot commited on Aug 6

Commit

2222481

verified ·

1 Parent(s): d6e2c02

Uploaded using `kernel-builder`.

Browse files

Files changed (28) hide show

benchmarks/benchmark.py +161 -0
build/torch-stable-abi29-cu128-x86_64-linux/__init__.py +24 -0
build/torch-stable-abi29-cu128-x86_64-linux/_nvfp4_gemm_cuda_9c5a0e3.abi3.so +3 -0
build/torch-stable-abi29-cu128-x86_64-linux/_ops.py +9 -0
build/torch-stable-abi29-cu128-x86_64-linux/_pack.py +110 -0
build/torch-stable-abi29-cu128-x86_64-linux/functional.py +75 -0
build/torch-stable-abi29-cu128-x86_64-linux/layers.py +215 -0
build/torch-stable-abi29-cu128-x86_64-linux/metadata.json +38 -0
build/torch-stable-abi29-cu128-x86_64-linux/metadata.json.sigstore +1 -0
build/torch-stable-abi29-cu128-x86_64-linux/nvfp4_gemm/__init__.py +26 -0
build/torch-stable-abi29-cu130-x86_64-linux/__init__.py +24 -0
build/torch-stable-abi29-cu130-x86_64-linux/_nvfp4_gemm_cuda_9c5a0e3.abi3.so +3 -0
build/torch-stable-abi29-cu130-x86_64-linux/_ops.py +9 -0
build/torch-stable-abi29-cu130-x86_64-linux/_pack.py +110 -0
build/torch-stable-abi29-cu130-x86_64-linux/functional.py +75 -0
build/torch-stable-abi29-cu130-x86_64-linux/layers.py +215 -0
build/torch-stable-abi29-cu130-x86_64-linux/metadata.json +38 -0
build/torch-stable-abi29-cu130-x86_64-linux/metadata.json.sigstore +1 -0
build/torch-stable-abi29-cu130-x86_64-linux/nvfp4_gemm/__init__.py +26 -0
build/torch-stable-abi29-cu132-x86_64-linux/__init__.py +24 -0
build/torch-stable-abi29-cu132-x86_64-linux/_nvfp4_gemm_cuda_9c5a0e3.abi3.so +3 -0
build/torch-stable-abi29-cu132-x86_64-linux/_ops.py +9 -0
build/torch-stable-abi29-cu132-x86_64-linux/_pack.py +110 -0
build/torch-stable-abi29-cu132-x86_64-linux/functional.py +75 -0
build/torch-stable-abi29-cu132-x86_64-linux/layers.py +215 -0
build/torch-stable-abi29-cu132-x86_64-linux/metadata.json +38 -0
build/torch-stable-abi29-cu132-x86_64-linux/metadata.json.sigstore +1 -0
build/torch-stable-abi29-cu132-x86_64-linux/nvfp4_gemm/__init__.py +26 -0

benchmarks/benchmark.py ADDED Viewed

	@@ -0,0 +1,161 @@

+import torch
+from kernels.benchmark import Benchmark
+SHAPES = {
+    "qkv": (8704, 6656),
+    "o": (6656, 4096),
+    "mlp": (39936, 6656),
+    "down": (6656, 19968),
+    "lm_head": (202048, 6656),
+}
+def _dequant(pw):
+    grid = torch.tensor(
+        [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0], device=pw.qweight.device
+    )
+    gs = pw.global_scale.float()
+    out = torch.empty(pw.n, pw.k, dtype=torch.float16, device=pw.qweight.device)
+    step = 16384
+    for i in range(0, pw.n, step):
+        qw = pw.qweight[i : i + step]
+        nib = torch.stack([qw & 0xF, qw >> 4], dim=-1)
+        nib = nib.reshape(qw.shape[0], pw.k).long()
+        sign = torch.where((nib & 0x8) != 0, -1.0, 1.0)
+        vals = grid[nib & 0x7] * sign
+        sf = pw.sf_rowmajor[i : i + step].view(torch.float8_e4m3fn).float()
+        scale = sf.repeat_interleave(16, dim=1)
+        out[i : i + step] = ((vals * scale) / gs).to(torch.float16)
+    return out
+class NvFP4GemmBenchmark(Benchmark):
+    seed: int = 42
+    def _setup_shape(self, name, m):
+        n, k = SHAPES[name]
+        torch.manual_seed(self.seed)
+        w = torch.randn(n, k, dtype=torch.bfloat16) * 0.02
+        self.pw = self.kernel.pack(w.to(self.device))
+        self.x = torch.randn(m, k, dtype=torch.bfloat16, device=self.device) * 0.125
+        self.ref_w = _dequant(self.pw)
+        self.xs = self.x.to(torch.float16)
+    def setup_decode_qkv(self):
+        self._setup_shape("qkv", 1)
+    def benchmark_decode_qkv(self):
+        self.out = self.kernel.gemm(self.pw, self.x)
+    def verify_decode_qkv(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_decode_o(self):
+        self._setup_shape("o", 1)
+    def benchmark_decode_o(self):
+        self.out = self.kernel.gemm(self.pw, self.x)
+    def verify_decode_o(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_decode_mlp(self):
+        self._setup_shape("mlp", 1)
+    def benchmark_decode_mlp(self):
+        self.out = self.kernel.gemm(self.pw, self.x)
+    def verify_decode_mlp(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_decode_down(self):
+        self._setup_shape("down", 1)
+    def benchmark_decode_down(self):
+        self.out = self.kernel.gemm(self.pw, self.x)
+    def verify_decode_down(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_decode_lm_head(self):
+        self._setup_shape("lm_head", 1)
+    def benchmark_decode_lm_head(self):
+        self.out = self.kernel.gemm(self.pw, self.x)
+    def verify_decode_lm_head(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_prefill_mlp(self):
+        self._setup_shape("mlp", 512)
+        self.x = self.x * 0.2
+        xq = self.kernel.quantize_reference(self.x.cpu()).to(self.device)
+        self.xs = xq.to(torch.float16)
+    def benchmark_prefill_mlp(self):
+        self.out = self.kernel.gemm(self.pw, self.x)
+    def verify_prefill_mlp(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+class NvFP4RawGemvBenchmark(Benchmark):
+    seed: int = 42
+    def _setup_shape(self, name):
+        n, k = SHAPES[name]
+        torch.manual_seed(self.seed)
+        w = torch.randn(n, k, dtype=torch.bfloat16) * 0.02
+        pw = self.kernel.pack(w.to(self.device))
+        self.qw = pw.qweight
+        self.sf = pw.sf_rowmajor
+        self.alpha = (1.0 / pw.global_scale).float().reshape(1)
+        self.x = torch.randn(1, k, dtype=torch.bfloat16, device=self.device) * 0.125
+        self.ref_w = _dequant(pw)
+        self.xs = self.x.to(torch.float16)
+    def setup_gemv_qkv(self):
+        self._setup_shape("qkv")
+    def benchmark_gemv_qkv(self):
+        self.out = self.kernel._ops.ops.nvfp4_gemv(self.x, self.qw, self.sf, self.alpha)
+    def verify_gemv_qkv(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_gemv_o(self):
+        self._setup_shape("o")
+    def benchmark_gemv_o(self):
+        self.out = self.kernel._ops.ops.nvfp4_gemv(self.x, self.qw, self.sf, self.alpha)
+    def verify_gemv_o(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_gemv_mlp(self):
+        self._setup_shape("mlp")
+    def benchmark_gemv_mlp(self):
+        self.out = self.kernel._ops.ops.nvfp4_gemv(self.x, self.qw, self.sf, self.alpha)
+    def verify_gemv_mlp(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_gemv_down(self):
+        self._setup_shape("down")
+    def benchmark_gemv_down(self):
+        self.out = self.kernel._ops.ops.nvfp4_gemv(self.x, self.qw, self.sf, self.alpha)
+    def verify_gemv_down(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)
+    def setup_gemv_lm_head(self):
+        self._setup_shape("lm_head")
+    def benchmark_gemv_lm_head(self):
+        self.out = self.kernel._ops.ops.nvfp4_gemv(self.x, self.qw, self.sf, self.alpha)
+    def verify_gemv_lm_head(self):
+        return (self.xs @ self.ref_w.T).to(torch.bfloat16)

build/torch-stable-abi29-cu128-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,24 @@

+from . import layers
+from .layers import fuse_decode_projections
+from .functional import gemm, gemm_swiglu
+from ._pack import (
+    PackedWeight,
+    global_scale_for,
+    pack,
+    pack_swiglu,
+    quantize_reference,
+    swizzled_sf_shape,
+)
+__all__ = [
+    "PackedWeight",
+    "pack",
+    "pack_swiglu",
+    "quantize_reference",
+    "global_scale_for",
+    "swizzled_sf_shape",
+    "gemm",
+    "gemm_swiglu",
+    "layers",
+    "fuse_decode_projections",
+]

build/torch-stable-abi29-cu128-x86_64-linux/_nvfp4_gemm_cuda_9c5a0e3.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:a57bde35bea7918bb524003d916e0074a3b72f5ed76a48c3fcdfe397ae65a8ba
+size 2987544

build/torch-stable-abi29-cu128-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _nvfp4_gemm_cuda_9c5a0e3
+ops = torch.ops._nvfp4_gemm_cuda_9c5a0e3
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_nvfp4_gemm_cuda_9c5a0e3::{op_name}"

build/torch-stable-abi29-cu128-x86_64-linux/_pack.py ADDED Viewed

	@@ -0,0 +1,110 @@

+from dataclasses import dataclass
+import torch
+from ._ops import ops
+FLOAT8_E4M3_MAX = 448.0
+FLOAT4_E2M1_MAX = 6.0
+SF_VEC_SIZE = 16  # elements sharing one FP8 scale
+def _round_up(x: int, m: int) -> int:
+    return (x + m - 1) // m * m
+def swizzled_sf_shape(rows: int, cols: int) -> tuple[int, int]:
+    """Shape of the swizzled scale-factor tensor (dtype int32) produced by
+    `scaled_fp4_quant(..., is_sf_swizzled_layout=True)` for a [rows, cols]
+    input. Mirrors `computeSwizzledSFShape` in nvfp4_utils.cuh: 128-row tiles,
+    4 FP8 scales packed per int32 column."""
+    return _round_up(rows, 128), _round_up(cols // SF_VEC_SIZE, 4) // 4
+def global_scale_for(w: torch.Tensor) -> torch.Tensor:
+    """Per-tensor second-level scale: FP8_MAX * FP4_MAX / amax."""
+    amax = w.abs().amax().to(torch.float32)
+    return (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / amax.clamp(min=1e-12)).reshape(1)
+@dataclass
+class PackedWeight:
+    """NVFP4 representation of a bf16 matrix [n, k]."""
+    qweight: torch.Tensor  # uint8 [n, k/2], two e2m1 values per byte
+    sf: torch.Tensor  # int32, swizzled FP8-E4M3 block scales
+    global_scale: torch.Tensor  # fp32 [1]
+    n: int
+    k: int
+    swiglu: bool = False
+    # uint8 [n, k/16] row-major FP8-E4M3 block scales, consumed by the W4A16
+    # GEMV decode path (None when only the swizzled layout is available).
+    sf_rowmajor: torch.Tensor | None = None
+    def to(self, device) -> "PackedWeight":
+        return PackedWeight(
+            qweight=self.qweight.to(device),
+            sf=self.sf.to(device),
+            global_scale=self.global_scale.to(device),
+            n=self.n,
+            k=self.k,
+            swiglu=self.swiglu,
+            sf_rowmajor=(
+                None if self.sf_rowmajor is None else self.sf_rowmajor.to(device)
+            ),
+        )
+def pack(
+    w: torch.Tensor, device: str | torch.device = "cuda", swiglu: bool = False
+) -> PackedWeight:
+    """Quantize a bf16/fp16 matrix [n, k] to NVFP4 (requires sm100+)."""
+    if w.dim() != 2 or not w.is_floating_point():
+        raise ValueError("pack() expects a 2-D floating-point weight")
+    n, k = w.shape
+    if k % SF_VEC_SIZE != 0:
+        raise ValueError(f"in_features must be a multiple of {SF_VEC_SIZE}, got {k}")
+    wd = w.detach().to(device, torch.bfloat16).contiguous()
+    gs = global_scale_for(wd).to(device)
+    qweight, sf = ops.scaled_fp4_quant(wd, gs, True)
+    # Row-major scales for the W4A16 GEMV decode path; the qweight output is
+    # identical to the swizzled call's, so it is discarded.
+    _, sf_rowmajor = ops.scaled_fp4_quant(wd, gs, False)
+    return PackedWeight(
+        qweight=qweight,
+        sf=sf,
+        global_scale=gs,
+        n=n,
+        k=k,
+        swiglu=swiglu,
+        sf_rowmajor=sf_rowmajor,
+    )
+def pack_swiglu(
+    w_gate: torch.Tensor, w_up: torch.Tensor, device: str | torch.device = "cuda"
+) -> PackedWeight:
+    """Pack gate and up projections [n, k] as one [2n, k] matrix; the GEMM
+    output has gate values in columns [0, n) and up in [n, 2n)."""
+    if w_gate.shape != w_up.shape:
+        raise ValueError("gate and up matrices must have the same shape")
+    return pack(torch.cat([w_gate, w_up], dim=0), device=device, swiglu=True)
+def quantize_reference(w: torch.Tensor) -> torch.Tensor:
+    """Pure-torch NVFP4 emulation (no GPU kernel): quantize + dequantize to
+    the same numerics the kernel produces, for tests and quality analysis."""
+    orig_dtype = w.dtype
+    wf = w.float()
+    gs = global_scale_for(wf)  # fp8_max*fp4_max/amax
+    n, k = wf.shape
+    g = wf.reshape(n, k // SF_VEC_SIZE, SF_VEC_SIZE)
+    # Block scale: amax/FP4_MAX * global_scale, stored in FP8-E4M3.
+    block_amax = g.abs().amax(-1, keepdim=True)
+    sf = (block_amax / FLOAT4_E2M1_MAX) * gs
+    sf = sf.to(torch.float8_e4m3fn).float().clamp(min=1e-12)
+    scaled = g * (gs / sf)
+    # e2m1 grid: {0, .5, 1, 1.5, 2, 3, 4, 6} with signs.
+    grid = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
+    q = grid[(scaled.abs().unsqueeze(-1) - grid).abs().argmin(-1)] * scaled.sign()
+    return (q * sf / gs).reshape(n, k).to(orig_dtype)

build/torch-stable-abi29-cu128-x86_64-linux/functional.py ADDED Viewed

	@@ -0,0 +1,75 @@

+import torch
+from ._ops import add_op_namespace_prefix, ops
+from ._pack import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, PackedWeight, swizzled_sf_shape
+@torch.library.register_fake(add_op_namespace_prefix("scaled_fp4_quant"))
+def _quant_fake(input, input_global_scale, is_sf_swizzled_layout):
+    n = input.size(-1)
+    m = input.numel() // n
+    out = input.new_empty((m, n // 2), dtype=torch.uint8)
+    if is_sf_swizzled_layout:
+        sf_m, sf_n = swizzled_sf_shape(m, n)
+        sf = input.new_empty((sf_m, sf_n), dtype=torch.int32)
+    else:
+        sf = input.new_empty((m, n // 16), dtype=torch.uint8)
+    return out, sf
+@torch.library.register_fake(add_op_namespace_prefix("cutlass_scaled_fp4_mm"))
+def _mm_fake(a, b, a_sf, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv"))
+def _gemv_fake(a, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv_swiglu"))
+def _gemv_swiglu_fake(a, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0) // 2), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv_gated"))
+def _gemv_gated_fake(a, g, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+def _as_fp8(sf: torch.Tensor) -> torch.Tensor:
+    return sf.view(torch.float8_e4m3fn) if sf.dtype == torch.int32 else sf
+def gemm(pw: PackedWeight, x: torch.Tensor) -> torch.Tensor:
+    """y[..., n] = x[..., k] @ W.T with dynamic NVFP4 activation quantization
+    (W4A4 on the Blackwell block-scaled tensor cores)."""
+    if x.shape[-1] != pw.k:
+        raise ValueError(f"expected {pw.k} input features, got {x.shape[-1]}")
+    x2 = x.reshape(-1, pw.k)
+    if not x2.is_contiguous():
+        x2 = x2.contiguous()
+    # Decode path: W4A16 GEMV keeps activations in bf16 (no activation
+    # quantization). The kernel re-reads the weight per output row, so it is
+    # only worthwhile for tiny m; prefill stays on the W4A4 CUTLASS path.
+    if x2.size(0) <= 2 and getattr(pw, "sf_rowmajor", None) is not None:
+        alpha = (1.0 / pw.global_scale).to(torch.float32).reshape(1)
+        y = ops.nvfp4_gemv(x2.to(torch.bfloat16), pw.qweight, pw.sf_rowmajor, alpha)
+        return y.reshape(*x.shape[:-1], pw.n)
+    a_amax = x2.abs().amax().to(torch.float32)
+    a_gs = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax.clamp(min=1e-12)).reshape(1)
+    aq, a_sf = ops.scaled_fp4_quant(x2.to(torch.bfloat16), a_gs, True)
+    alpha = (1.0 / (a_gs * pw.global_scale)).to(torch.float32)
+    # The quant op returns swizzled scales packed 4-per-int32; the GEMM takes
+    # the same bytes typed as fp8-e4m3.
+    y = ops.cutlass_scaled_fp4_mm(aq, pw.qweight, _as_fp8(a_sf), _as_fp8(pw.sf), alpha)
+    return y.reshape(*x.shape[:-1], pw.n)
+def gemm_swiglu(pw: PackedWeight, x: torch.Tensor) -> torch.Tensor:
+    """silu(x @ W_gate.T) * (x @ W_up.T) for a SwiGLU-packed matrix."""
+    if not pw.swiglu:
+        raise ValueError("packed matrix was not packed with pack_swiglu")
+    y = gemm(pw, x)
+    half = pw.n // 2
+    return torch.nn.functional.silu(y[..., :half]) * y[..., half:]

build/torch-stable-abi29-cu128-x86_64-linux/layers.py ADDED Viewed

	@@ -0,0 +1,215 @@

+import torch
+import torch.nn as nn
+from .functional import gemm
+from ._ops import ops
+from ._pack import PackedWeight
+class NVFP4Linear(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(self, x):
+        pw = PackedWeight(
+            qweight=self.weight,
+            sf=self.weight_sf,
+            global_scale=self.weight_global_scale,
+            n=self.out_features,
+            k=self.in_features,
+            sf_rowmajor=getattr(self, "weight_sf_rowmajor", None),
+        )
+        return gemm(pw, x)
+def _ntokens(x):
+    return x.numel() // x.shape[-1]
+def _fused_gemv(module, x):
+    return ops.nvfp4_gemv(
+        x.reshape(-1, x.shape[-1]).to(torch.bfloat16),
+        module.nvfp4_fused_qweight,
+        module.nvfp4_fused_sf,
+        module.nvfp4_fused_alpha,
+    )
+class NVFP4FusedSwiGLUMLP(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(self, x):
+        if _ntokens(x) <= 2 and hasattr(self, "nvfp4_fused_qweight"):
+            x2 = x.reshape(-1, x.shape[-1]).to(torch.bfloat16)
+            h = ops.nvfp4_gemv_swiglu(
+                x2,
+                self.nvfp4_fused_qweight,
+                self.nvfp4_fused_sf,
+                self.nvfp4_fused_alpha,
+            )
+            return self.down_proj(h.reshape(*x.shape[:-1], -1))
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+class NVFP4GatedAttention(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(
+        self,
+        hidden_states,
+        position_embeddings=None,
+        attention_mask=None,
+        past_key_values=None,
+        **kwargs,
+    ):
+        from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+        og = None
+        if _ntokens(hidden_states) <= 2 and hasattr(self, "nvfp4_fused_qweight"):
+            fused = _fused_gemv(self, hidden_states)
+            q, k, v, *rest = torch.split(fused, self.nvfp4_fused_splits, dim=-1)
+            og = rest[0] if rest else None
+        else:
+            q = self.q_proj(hidden_states)
+            k = self.k_proj(hidden_states)
+            v = self.v_proj(hidden_states)
+        query_states = q.reshape(hidden_shape).transpose(1, 2)
+        key_states = k.reshape(hidden_shape).transpose(1, 2)
+        value_states = v.reshape(hidden_shape).transpose(1, 2)
+        norm = getattr(self, "qk_norm", None)
+        if norm is not None:
+            query_states = norm(query_states) * getattr(self, "qk_scale_factor", 1.0)
+            key_states = norm(key_states)
+        if position_embeddings is not None:
+            cos, sin = position_embeddings
+            query_states, key_states = self.nvfp4_rope(
+                query_states, key_states, cos, sin
+            )
+        if past_key_values is not None:
+            key_states, value_states = past_key_values.update(
+                key_states, value_states, self.layer_idx
+            )
+        attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, self.nvfp4_eager
+        )
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=self.attention_dropout if self.training else 0.0,
+            scaling=self.scaling,
+            sliding_window=getattr(self, "sliding_window", None),
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        if og is None and getattr(self, "gate_proj", None) is not None:
+            og = self.gate_proj(hidden_states)
+        o = self.o_proj
+        if og is not None and _ntokens(attn_output) <= 2 and _has_nvfp4(o):
+            a2 = attn_output.reshape(-1, attn_output.shape[-1])
+            og2 = og.reshape(a2.shape).contiguous()
+            alpha = (1.0 / o.weight_global_scale.float()).reshape(1)
+            y = ops.nvfp4_gemv_gated(a2, og2, o.weight, o.weight_sf_rowmajor, alpha)
+            return y.reshape(*input_shape, -1), attn_weights
+        if og is not None:
+            attn_output = attn_output * torch.sigmoid(og.reshape(attn_output.shape))
+        return o(attn_output), attn_weights
+def _has_nvfp4(m):
+    return m is not None and hasattr(m, "weight_sf_rowmajor")
+def _is_silu(act):
+    return isinstance(act, nn.SiLU) or type(act).__name__ == "SiLUActivation"
+def _attach_fused(module, mods):
+    # Keep a single packed allocation and expose row views back through the
+    # original projection modules.  Previously fusion allocated concatenated
+    # weights while retaining an independent copy in every projection.
+    # Attention and SwiGLU use row-concatenated storage; the latter is handled
+    # by the GEMV kernel using the gate/up half offset.
+    fused_qweight = torch.cat([m.weight for m in mods], 0).contiguous()
+    fused_sf = torch.cat([m.weight_sf_rowmajor for m in mods], 0).contiguous()
+    module.nvfp4_fused_qweight = fused_qweight
+    module.nvfp4_fused_sf = fused_sf
+    module.nvfp4_fused_alpha = torch.cat(
+        [
+            (1.0 / m.weight_global_scale.float())
+            .reshape(1)
+            .expand(m.out_features)
+            .contiguous()
+            for m in mods
+        ]
+    )
+    module.nvfp4_fused_splits = [m.out_features for m in mods]
+    offset = 0
+    for m in mods:
+        end = offset + m.out_features
+        m.weight = fused_qweight[offset:end]
+        m.weight_sf_rowmajor = fused_sf[offset:end]
+        offset = end
+def _capture_helpers(cls, rope_fn):
+    if getattr(cls, "nvfp4_eager", None) is not None:
+        return
+    fwd = cls.forward
+    names = ("apply_rotary_pos_emb_interleave", "apply_rotary_pos_emb")
+    rope = rope_fn or next(
+        (fwd.__globals__[n] for n in names if n in fwd.__code__.co_names), None
+    )
+    cls.nvfp4_rope = staticmethod(rope) if rope is not None else None
+    cls.nvfp4_eager = staticmethod(fwd.__globals__["eager_attention_forward"])
+def _mark(cls, layer_name):
+    if getattr(cls, "kernel_layer_name", None) is not None:
+        return
+    try:
+        from kernels import use_kernel_forward_from_hub
+        use_kernel_forward_from_hub(layer_name)(cls)
+    except Exception:
+        cls.kernel_layer_name = layer_name
+def fuse_decode_projections(model, rope_fn=None):
+    n = 0
+    for mod in model.modules():
+        qkv = [getattr(mod, p, None) for p in ("q_proj", "k_proj", "v_proj")]
+        if (
+            all(_has_nvfp4(p) for p in qkv)
+            and hasattr(mod, "o_proj")
+            and hasattr(mod, "layer_idx")
+        ):
+            gate = getattr(mod, "gate_proj", None)
+            _attach_fused(mod, qkv + ([gate] if _has_nvfp4(gate) else []))
+            _capture_helpers(type(mod), rope_fn)
+            _mark(type(mod), "NVFP4GatedAttention")
+            n += 1
+            continue
+        gu = [getattr(mod, p, None) for p in ("gate_proj", "up_proj")]
+        if (
+            all(_has_nvfp4(p) for p in gu)
+            and hasattr(mod, "down_proj")
+            and _is_silu(getattr(mod, "act_fn", None))
+            and gu[0].out_features == gu[1].out_features
+        ):
+            _attach_fused(mod, gu)
+            _mark(type(mod), "NVFP4SwiGLUMLP")
+            n += 1
+    return n

build/torch-stable-abi29-cu128-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "nvfp4-gemm",
+  "id": "_nvfp4_gemm_cuda_9c5a0e3",
+  "version": 1,
+  "license": "Apache-2.0",
+  "upstream": "git+https://github.com/vllm-project/vllm.git",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0a",
+      "12.0a"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "ARXik9k13ZmZ/1tDxHgH6bQyJ8Cn6wmCrz5WEM+DABI=",
+      "_nvfp4_gemm_cuda_9c5a0e3.abi3.so": "pXveNb6nkYu1JAA9kW4AdKO3L17XakjD/N/jl65lqLo=",
+      "_ops.py": "+jBXp3Uoatvj6U0lcsYwN/3KdSd8+Aknhm5T+xbff6w=",
+      "_pack.py": "jATF2LhpSLUEiPWSrhsCWeyxIyn80CSy2QJNkKMADlc=",
+      "functional.py": "ZTd0hcUOkDdU9ZvpQMPC81KBP3FUvkmfZu7d7mhZRss=",
+      "layers.py": "asplwNUY3+drs57APjw3J9kFLtswK9YI0SxRVHZj2Hg=",
+      "nvfp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "9c5a0e35e9733ee93d458d2d4d5814f67d209d29",
+      "dirty": false
+    }
+  }
+}

build/torch-stable-abi29-cu128-x86_64-linux/metadata.json.sigstore ADDED Viewed

	@@ -0,0 +1 @@

+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHSjCCBtGgAwIBAgIUS9OSD73Crs3cQlv9jdxLWV7a4AEwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYwODA2MTQwMzQ2WhcNMjYwODA2MTQxMzQ2WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE7gZqHDNOMOz4Q8FeyZVlihIXU9AvTzg6euBRbzyxDrLTmcVUvJu3N7RJKMEan5CwIH0OALdYikglKoQ2vkhd2KOCBfAwggXsMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQU5z6L3FGXVChZAWCu0GEujVULvPswHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wawYDVR0RAQH/BGEwX4ZdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDkGCisGAQQBg78wAQEEK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wHwYKKwYBBAGDvzABAgQRd29ya2Zsb3dfZGlzcGF0Y2gwNgYKKwYBBAGDvzABAwQoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTATBgorBgEEAYO/MAEEBAVCdWlsZDArBgorBgEEAYO/MAEFBB1odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTAdBgorBgEEAYO/MAEGBA9yZWZzL2hlYWRzL21haW4wOwYKKwYBBAGDvzABCAQtDCtodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMG0GCisGAQQBg78wAQkEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wAQoEKgwoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTAbBgorBgEEAYO/MAELBA0MC3NlbGYtaG9zdGVkMEAGCisGAQQBg78wAQwEMgwwaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MDgGCisGAQQBg78wAQ0EKgwoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTAfBgorBgEEAYO/MAEOBBEMD3JlZnMvaGVhZHMvbWFpbjAaBgorBgEEAYO/MAEPBAwMCjEwNzE0NzU1MjkwLgYKKwYBBAGDvzABEAQgDB5odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2UwGAYKKwYBBAGDvzABEQQKDAgyNTcyMDc0MzBtBgorBgEEAYO/MAESBF8MXWh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eS8uZ2l0aHViL3dvcmtmbG93cy9idWlsZC55YW1sQHJlZnMvaGVhZHMvbWFpbjA4BgorBgEEAYO/MAETBCoMKDljNWEwZTM1ZTk3MzNlZTkzZDQ1OGQyZDRkNTgxNGY2N2QyMDlkMjkwIQYKKwYBBAGDvzABFAQTDBF3b3JrZmxvd19kaXNwYXRjaDBkBgorBgEEAYO/MAEVBFYMVGh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eS9hY3Rpb25zL3J1bnMvMzExMDgyOTgwNDUvYXR0ZW1wdHMvMTAWBgorBgEEAYO/MAEWBAgMBnB1YmxpYzBGBgorBgEEAYO/MAEYBDgMNnJlcG86aHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHk6cmVmOnJlZnMvaGVhZHMvbWFpbjCBigYKKwYBBAHWeQIEAgR8BHoAeAB2AN09MGrGxxEyYxkeHJlnNwKiSl643jyt/4eKcoAvKe6OAAABn9djIroAAAQDAEcwRQIgEpfpnLDesOAQNDUp8m0yF2wvPAFoeOdx9ISDqtauwoMCIQDQatHRh59ieCHLBGQUKrZ+itQ4BYRDw8Iuc68iIN2yqTAKBggqhkjOPQQDAwNnADBkAjB9q9YrxbAcKxC6YxC3iIS5TTtGqrD/0p6yXCn1Rfbj8sVqp2y6XqzqCw27XDOr+/sCMBDJIfFTbRoyAo1CeXvtzjVsg1kyq/coxBG0OYe0MRhCQxi1GlXLNpTgUeljDF5PPA=="}, "tlogEntries":[{"logIndex":"2360822422", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1786025026", "inclusionPromise":{"signedEntryTimestamp":"MEYCIQD4uRzsrIjcaXHD1riGjwx70gmfNhYNSSKad+gGnL48FwIhAPC/ipDCJKDDJ3faJqNNzxD9ojhpTY7ks80diyGkiA74"}, "inclusionProof":{"logIndex":"2238918160", "rootHash":"Inbkmbis8U2sMiEM6EmTLZwA5s7x9Sik3HRcVtFMZsM=", "treeSize":"2238918170", "hashes":["DLvWUnbgILnx2+gOnJhcAMtoy/dgm6AFYaJFtiwG8Zg=", "zrSnDQOKLAZKUSbtZvNdr806Ret7zVdbR/kbIw8+d6g=", "yIxZWohpF6TmhojqaHkM2V5308xv0dTGBpt62aZIndE=", "s22eUbTXqsYuVO1WL518IVbaqZCnT/gJ8+Cy7ialgsU=", "Boblyud220auq3IRgfn7lwa+B1whTYwPKuWoUUCiG5w=", "yOi1SZCAJyQeGFNfXEyJmZfgE/yHiViHJh7AkVStYx0=", "ZDDgBLJeZMIedbwsKMeIdE+rnhT6LFyxK01q4RQswB8=", "1QfEqQEF22LcM0N+z5i77k0Oy00Arz4yjX7D1LTJP34=", "tSch9ErXVKdLoCFPSZ8D8aLt8T+YHXh/0cV7rQ+HxWc=", "VN19Pd6HJSN2xUEEfXlAK+NBL7uztCXLEaUeuJxgAbY=", "2DzQwQqFGfdTX3BVYQkcDxiT/TBV7lnr+ya29yCNm+o=", "QVx27ou3XJyzOgzVsHRov+ie3M7wOfT492kpmkbAeBo=", "lLhES7her3Dzu/HSMpPVolTSfVQlKbD23EIsEwXq4V4=", "kVyfjHB6dSBvAqnJBVxvTpb/VPDK9zzq5/txahxnbhc=", "u23nWGlBSoQOMevXp72NpPaDjuN0qcB01i17Odcs3Wk=", "i5Zl8FZrDwxCDv2e2DNO2M8JvpR/c11ElvCZS53/teA=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n2238918170\nInbkmbis8U2sMiEM6EmTLZwA5s7x9Sik3HRcVtFMZsM=\n\n— rekor.sigstore.dev wNI9ajBEAiAN94fxCmq+jARLavCZelggw0xfIsLdrUI02+C1Z6/LsgIgMV4fswAMOmC7Iztunae8RAFJKZhsGEde4JBU7ziBs9s=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIyMWJlOWNlMzc3MWUzN2U3MDI3MTk1ZTRkMDMyNDc0MGY0ZWQyMTM0MTE1NzA0MzI0YThkNmM2OTliNDMwMTQyIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FVUNJUUNVeXhrdFFjQWJUZ1ovSjRsRnhXY2hLbHFaWU1kTUY1VUM3SHRQYmNSckRBSWdSdjlaOFZaVE9vTDQ0cldURW51c0h3cU9tWmdrSFg4b2lkUEFGd0Nub0NNPSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaFRha05EUW5SSFowRjNTVUpCWjBsVlV6bFBVMFEzTTBOeWN6TmpVV3gyT1dwa2VFeFhWamRoTkVGRmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFpkMDlFUVRKTlZGRjNUWHBSTWxkb1kwNU5hbGwzVDBSQk1rMVVVWGhOZWxFeVYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVUzWjFweFNFUk9UMDFQZWpSUk9FWmxlVnBXYkdsb1NWaFZPVUYyVkhwbk5tVjFRbElLWW5wNWVFUnlURlJ0WTFaVmRrcDFNMDQzVWtwTFRVVmhialZEZDBsSU1FOUJUR1JaYVd0bmJFdHZVVEoyYTJoa01rdFBRMEptUVhkbloxaHpUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXUkNaMDVXU0ZFMFJVWm5VVlUxZWpaTUNqTkdSMWhXUTJoYVFWZERkVEJIUlhWcVZsVk1kbEJ6ZDBoM1dVUldVakJxUWtKbmQwWnZRVlV6T1ZCd2VqRlphMFZhWWpWeFRtcHdTMFpYYVhocE5Ga0tXa1E0ZDJGM1dVUldVakJTUVZGSUwwSkhSWGRZTkZwa1lVaFNNR05JVFRaTWVUbHVZVmhTYjJSWFNYVlpNamwwVERKb01Wb3laSEJpYldSdFdWZE9iQXBNTW5Sc1kyMDFiR0pJVFhSWk1qbDBZbGhXZFdGWVVqVk1lVFZ1WVZoU2IyUlhTWFprTWpsNVlUSmFjMkl6WkhwTU1rb3hZVmQ0YTB4dWJHaGlWM2hCQ21OdFZtMWplVGx2V2xkR2EyTjVPWFJaVjJ4MVRVUnJSME5wYzBkQlVWRkNaemM0ZDBGUlJVVkxNbWd3WkVoQ2VrOXBPSFprUnpseVdsYzBkVmxYVGpBS1lWYzVkV041Tlc1aFdGSnZaRmRLTVdNeVZubFpNamwxWkVkV2RXUkROV3BpTWpCM1NIZFpTMHQzV1VKQ1FVZEVkbnBCUWtGblVWSmtNamw1WVRKYWN3cGlNMlJtV2tkc2VtTkhSakJaTW1kM1RtZFpTMHQzV1VKQ1FVZEVkbnBCUWtGM1VXOVBWMDB4V1ZSQ2JFMTZWbXhQVkdONlRUSldiRTlVVG10T1JGVTBDbHBFU210T1IxRXhUMFJGTUZwcVdUTmFSRWwzVDFkUmVVOVVRVlJDWjI5eVFtZEZSVUZaVHk5TlFVVkZRa0ZXUTJSWGJITmFSRUZ5UW1kdmNrSm5SVVVLUVZsUEwwMUJSVVpDUWpGdlpGZGtibUZYTlc1YWJVWnFXbE01Y2xwWVNuVmFWM2g2VEZkT2RtSlhNVEZpYld3d1pWUkJaRUpuYjNKQ1owVkZRVmxQTHdwTlFVVkhRa0U1ZVZwWFducE1NbWhzV1ZkU2Vrd3lNV2hoVnpSM1QzZFpTMHQzV1VKQ1FVZEVkbnBCUWtOQlVYUkVRM1J2WkVoU2QyTjZiM1pNTTFKMkNtRXlWblZNYlVacVpFZHNkbUp1VFhWYU1td3dZVWhXYVdSWVRteGpiVTUyWW01U2JHSnVVWFZaTWpsMFRVY3dSME5wYzBkQlVWRkNaemM0ZDBGUmEwVUtXSGQ0WkdGSVVqQmpTRTAyVEhrNWJtRllVbTlrVjBsMVdUSTVkRXd5YURGYU1tUndZbTFrYlZsWFRteE1NblJzWTIwMWJHSklUWFJaTWpsMFlsaFdkUXBoV0ZJMVRIazFibUZZVW05a1YwbDJaREk1ZVdFeVduTmlNMlI2VERKS01XRlhlR3RNYm14b1lsZDRRV050Vm0xamVUbHZXbGRHYTJONU9YUlpWMngxQ2sxRVowZERhWE5IUVZGUlFtYzNPSGRCVVc5RlMyZDNiMDlYVFRGWlZFSnNUWHBXYkU5VVkzcE5NbFpzVDFST2EwNUVWVFJhUkVwclRrZFJNVTlFUlRBS1dtcFpNMXBFU1hkUFYxRjVUMVJCWWtKbmIzSkNaMFZGUVZsUEwwMUJSVXhDUVRCTlF6Tk9iR0pIV1hSaFJ6bDZaRWRXYTAxRlFVZERhWE5IUVZGUlFncG5OemgzUVZGM1JVMW5kM2RoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlUxRVowZERhWE5IUVZGUlFtYzNPSGRCVVRCRlMyZDNiMDlYVFRGWlZFSnNUWHBXYkU5VVkzcE5NbFpzVDFST2EwNUVWVFFLV2tSS2EwNUhVVEZQUkVVd1dtcFpNMXBFU1hkUFYxRjVUMVJCWmtKbmIzSkNaMFZGUVZsUEwwMUJSVTlDUWtWTlJETktiRnB1VFhaaFIxWm9Xa2hOZGdwaVYwWndZbXBCWVVKbmIzSkNaMFZGUVZsUEwwMUJSVkJDUVhkTlEycEZkMDU2UlRCT2VsVXhUV3ByZDB4bldVdExkMWxDUWtGSFJIWjZRVUpGUVZGbkNrUkNOVzlrU0ZKM1kzcHZka3d5WkhCa1IyZ3hXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWM1IwRlpTMHQzV1VKQ1FVZEVkbnBCUWtWUlVVc0tSRUZuZVU1VVkzbE5SR013VFhwQ2RFSm5iM0pDWjBWRlFWbFBMMDFCUlZOQ1JqaE5XRmRvTUdSSVFucFBhVGgyV2pKc01HRklWbWxNYlU1MllsTTVid3BrVjJSdVlWYzFibHB0Um1wYVV6bHlXbGhLZFZwWGVIcE1WMDUyWWxjeE1XSnRiREJsVXpoMVdqSnNNR0ZJVm1sTU0yUjJZMjEwYldKSE9UTmplVGxwQ21SWGJITmFRelUxV1ZjeGMxRklTbXhhYmsxMllVZFdhRnBJVFhaaVYwWndZbXBCTkVKbmIzSkNaMFZGUVZsUEwwMUJSVlJDUTI5TlMwUnNhazVYUlhjS1dsUk5NVnBVYXpOTmVrNXNXbFJyZWxwRVVURlBSMUY1V2tSU2EwNVVaM2hPUjFreVRqSlJlVTFFYkd0TmFtdDNTVkZaUzB0M1dVSkNRVWRFZG5wQlFncEdRVkZVUkVKR00ySXpTbkphYlhoMlpERTVhMkZZVG5kWldGSnFZVVJDYTBKbmIzSkNaMFZGUVZsUEwwMUJSVlpDUmxsTlZrZG9NR1JJUW5wUGFUaDJDbG95YkRCaFNGWnBURzFPZG1KVE9XOWtWMlJ1WVZjMWJscHRSbXBhVXpseVdsaEtkVnBYZUhwTVYwNTJZbGN4TVdKdGJEQmxVemxvV1ROU2NHSXlOWG9LVEROS01XSnVUWFpOZWtWNFRVUm5lVTlVWjNkT1JGVjJXVmhTTUZwWE1YZGtTRTEyVFZSQlYwSm5iM0pDWjBWRlFWbFBMMDFCUlZkQ1FXZE5RbTVDTVFwWmJYaHdXWHBDUjBKbmIzSkNaMFZGUVZsUEwwMUJSVmxDUkdkTlRtNUtiR05IT0RaaFNGWnVXakpzZFZveVdtaFpNbFYyWVRKV2VXSnRWbk5qZVRGcUNtSXlNWFJrVnpWd1pFaHJObU50Vm0xUGJrcHNXbTVOZG1GSFZtaGFTRTEyWWxkR2NHSnFRMEpwWjFsTFMzZFpRa0pCU0ZkbFVVbEZRV2RTT0VKSWIwRUtaVUZDTWtGT01EbE5SM0pIZUhoRmVWbDRhMlZJU214dVRuZExhVk5zTmpRemFubDBMelJsUzJOdlFYWkxaVFpQUVVGQlFtNDVaR3BKY205QlFVRlJSQXBCUldOM1VsRkpaMFZ3Wm5CdVRFUmxjMDlCVVU1RVZYQTRiVEI1UmpKM2RsQkJSbTlsVDJSNE9VbFRSSEYwWVhWM2IwMURTVkZFVVdGMFNGSm9OVGxwQ21WRFNFeENSMUZWUzNKYUsybDBVVFJDV1ZKRWR6aEpkV00yT0dsSlRqSjVjVlJCUzBKblozRm9hMnBQVUZGUlJFRjNUbTVCUkVKclFXcENPWEU1V1hJS2VHSkJZMHQ0UXpaWmVFTXphVWxUTlZSVWRFZHhja1F2TUhBMmVWaERiakZTWm1KcU9ITldjWEF5ZVRaWWNYcHhRM2N5TjFoRVQzSXJMM05EVFVKRVNncEpaa1pVWWxKdmVVRnZNVU5sV0haMGVtcFdjMmN4YTNseEwyTnZlRUpITUU5WlpUQk5VbWhEVVhocE1VZHNXRXhPY0ZSblZXVnNha1JHTlZCUVFUMDlDaTB0TFMwdFJVNUVJRU5GVWxSSlJrbERRVlJGTFMwdExTMEsifX19fQ=="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyDADAgEAMIICvwYJKoZIhvcNAQcCoIICsDCCAqwCAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgw1pVzNBcFREuotVjBVq4ac24HV/5E7bLCQe5t5jIi0gCFEp1QvjmRM06C3vt6D/r2EYGK947GA8yMDI2MDgwNjE0MDM0NlowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdowggHWAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYwODA2MTQwMzQ2WjAvBgkqhkiG9w0BCQQxIgQg2a7o4bl9utb9zteMoi/0SUUnf8lryCUpvcOKki8hxlcwgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGYwZAIwILnROv5qsr9Nt/eVheYqzE1snFA0UyPg7whadKIheNnLy5UxYH/cif1PiRxaYjT1AjA3BTNDpZ4M2k5OpGmoGhYhVmz2t6tQD+ZLpf7+oMmrPVk2zY13GDeBvTcrPCkW2og="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"Ib6c43ceN+cCcZXk0DJHQPTtITQRVwQySo1saZtDAUI="}, "signature":"MEUCIQCUyxktQcAbTgZ/J4lFxWchKlqZYMdMF5UC7HtPbcRrDAIgRv9Z8VZTOoL44rWTEnusHwqOmZgkHX8oidPAFwCnoCM="}}

build/torch-stable-abi29-cu128-x86_64-linux/nvfp4_gemm/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch-stable-abi29-cu130-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,24 @@

+from . import layers
+from .layers import fuse_decode_projections
+from .functional import gemm, gemm_swiglu
+from ._pack import (
+    PackedWeight,
+    global_scale_for,
+    pack,
+    pack_swiglu,
+    quantize_reference,
+    swizzled_sf_shape,
+)
+__all__ = [
+    "PackedWeight",
+    "pack",
+    "pack_swiglu",
+    "quantize_reference",
+    "global_scale_for",
+    "swizzled_sf_shape",
+    "gemm",
+    "gemm_swiglu",
+    "layers",
+    "fuse_decode_projections",
+]

build/torch-stable-abi29-cu130-x86_64-linux/_nvfp4_gemm_cuda_9c5a0e3.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:677d4daa14831cb173a766cd1c00ad62e33acffd66504685d841679a53cbd315
+size 3124488

build/torch-stable-abi29-cu130-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _nvfp4_gemm_cuda_9c5a0e3
+ops = torch.ops._nvfp4_gemm_cuda_9c5a0e3
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_nvfp4_gemm_cuda_9c5a0e3::{op_name}"

build/torch-stable-abi29-cu130-x86_64-linux/_pack.py ADDED Viewed

	@@ -0,0 +1,110 @@

+from dataclasses import dataclass
+import torch
+from ._ops import ops
+FLOAT8_E4M3_MAX = 448.0
+FLOAT4_E2M1_MAX = 6.0
+SF_VEC_SIZE = 16  # elements sharing one FP8 scale
+def _round_up(x: int, m: int) -> int:
+    return (x + m - 1) // m * m
+def swizzled_sf_shape(rows: int, cols: int) -> tuple[int, int]:
+    """Shape of the swizzled scale-factor tensor (dtype int32) produced by
+    `scaled_fp4_quant(..., is_sf_swizzled_layout=True)` for a [rows, cols]
+    input. Mirrors `computeSwizzledSFShape` in nvfp4_utils.cuh: 128-row tiles,
+    4 FP8 scales packed per int32 column."""
+    return _round_up(rows, 128), _round_up(cols // SF_VEC_SIZE, 4) // 4
+def global_scale_for(w: torch.Tensor) -> torch.Tensor:
+    """Per-tensor second-level scale: FP8_MAX * FP4_MAX / amax."""
+    amax = w.abs().amax().to(torch.float32)
+    return (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / amax.clamp(min=1e-12)).reshape(1)
+@dataclass
+class PackedWeight:
+    """NVFP4 representation of a bf16 matrix [n, k]."""
+    qweight: torch.Tensor  # uint8 [n, k/2], two e2m1 values per byte
+    sf: torch.Tensor  # int32, swizzled FP8-E4M3 block scales
+    global_scale: torch.Tensor  # fp32 [1]
+    n: int
+    k: int
+    swiglu: bool = False
+    # uint8 [n, k/16] row-major FP8-E4M3 block scales, consumed by the W4A16
+    # GEMV decode path (None when only the swizzled layout is available).
+    sf_rowmajor: torch.Tensor | None = None
+    def to(self, device) -> "PackedWeight":
+        return PackedWeight(
+            qweight=self.qweight.to(device),
+            sf=self.sf.to(device),
+            global_scale=self.global_scale.to(device),
+            n=self.n,
+            k=self.k,
+            swiglu=self.swiglu,
+            sf_rowmajor=(
+                None if self.sf_rowmajor is None else self.sf_rowmajor.to(device)
+            ),
+        )
+def pack(
+    w: torch.Tensor, device: str | torch.device = "cuda", swiglu: bool = False
+) -> PackedWeight:
+    """Quantize a bf16/fp16 matrix [n, k] to NVFP4 (requires sm100+)."""
+    if w.dim() != 2 or not w.is_floating_point():
+        raise ValueError("pack() expects a 2-D floating-point weight")
+    n, k = w.shape
+    if k % SF_VEC_SIZE != 0:
+        raise ValueError(f"in_features must be a multiple of {SF_VEC_SIZE}, got {k}")
+    wd = w.detach().to(device, torch.bfloat16).contiguous()
+    gs = global_scale_for(wd).to(device)
+    qweight, sf = ops.scaled_fp4_quant(wd, gs, True)
+    # Row-major scales for the W4A16 GEMV decode path; the qweight output is
+    # identical to the swizzled call's, so it is discarded.
+    _, sf_rowmajor = ops.scaled_fp4_quant(wd, gs, False)
+    return PackedWeight(
+        qweight=qweight,
+        sf=sf,
+        global_scale=gs,
+        n=n,
+        k=k,
+        swiglu=swiglu,
+        sf_rowmajor=sf_rowmajor,
+    )
+def pack_swiglu(
+    w_gate: torch.Tensor, w_up: torch.Tensor, device: str | torch.device = "cuda"
+) -> PackedWeight:
+    """Pack gate and up projections [n, k] as one [2n, k] matrix; the GEMM
+    output has gate values in columns [0, n) and up in [n, 2n)."""
+    if w_gate.shape != w_up.shape:
+        raise ValueError("gate and up matrices must have the same shape")
+    return pack(torch.cat([w_gate, w_up], dim=0), device=device, swiglu=True)
+def quantize_reference(w: torch.Tensor) -> torch.Tensor:
+    """Pure-torch NVFP4 emulation (no GPU kernel): quantize + dequantize to
+    the same numerics the kernel produces, for tests and quality analysis."""
+    orig_dtype = w.dtype
+    wf = w.float()
+    gs = global_scale_for(wf)  # fp8_max*fp4_max/amax
+    n, k = wf.shape
+    g = wf.reshape(n, k // SF_VEC_SIZE, SF_VEC_SIZE)
+    # Block scale: amax/FP4_MAX * global_scale, stored in FP8-E4M3.
+    block_amax = g.abs().amax(-1, keepdim=True)
+    sf = (block_amax / FLOAT4_E2M1_MAX) * gs
+    sf = sf.to(torch.float8_e4m3fn).float().clamp(min=1e-12)
+    scaled = g * (gs / sf)
+    # e2m1 grid: {0, .5, 1, 1.5, 2, 3, 4, 6} with signs.
+    grid = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
+    q = grid[(scaled.abs().unsqueeze(-1) - grid).abs().argmin(-1)] * scaled.sign()
+    return (q * sf / gs).reshape(n, k).to(orig_dtype)

build/torch-stable-abi29-cu130-x86_64-linux/functional.py ADDED Viewed

	@@ -0,0 +1,75 @@

+import torch
+from ._ops import add_op_namespace_prefix, ops
+from ._pack import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, PackedWeight, swizzled_sf_shape
+@torch.library.register_fake(add_op_namespace_prefix("scaled_fp4_quant"))
+def _quant_fake(input, input_global_scale, is_sf_swizzled_layout):
+    n = input.size(-1)
+    m = input.numel() // n
+    out = input.new_empty((m, n // 2), dtype=torch.uint8)
+    if is_sf_swizzled_layout:
+        sf_m, sf_n = swizzled_sf_shape(m, n)
+        sf = input.new_empty((sf_m, sf_n), dtype=torch.int32)
+    else:
+        sf = input.new_empty((m, n // 16), dtype=torch.uint8)
+    return out, sf
+@torch.library.register_fake(add_op_namespace_prefix("cutlass_scaled_fp4_mm"))
+def _mm_fake(a, b, a_sf, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv"))
+def _gemv_fake(a, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv_swiglu"))
+def _gemv_swiglu_fake(a, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0) // 2), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv_gated"))
+def _gemv_gated_fake(a, g, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+def _as_fp8(sf: torch.Tensor) -> torch.Tensor:
+    return sf.view(torch.float8_e4m3fn) if sf.dtype == torch.int32 else sf
+def gemm(pw: PackedWeight, x: torch.Tensor) -> torch.Tensor:
+    """y[..., n] = x[..., k] @ W.T with dynamic NVFP4 activation quantization
+    (W4A4 on the Blackwell block-scaled tensor cores)."""
+    if x.shape[-1] != pw.k:
+        raise ValueError(f"expected {pw.k} input features, got {x.shape[-1]}")
+    x2 = x.reshape(-1, pw.k)
+    if not x2.is_contiguous():
+        x2 = x2.contiguous()
+    # Decode path: W4A16 GEMV keeps activations in bf16 (no activation
+    # quantization). The kernel re-reads the weight per output row, so it is
+    # only worthwhile for tiny m; prefill stays on the W4A4 CUTLASS path.
+    if x2.size(0) <= 2 and getattr(pw, "sf_rowmajor", None) is not None:
+        alpha = (1.0 / pw.global_scale).to(torch.float32).reshape(1)
+        y = ops.nvfp4_gemv(x2.to(torch.bfloat16), pw.qweight, pw.sf_rowmajor, alpha)
+        return y.reshape(*x.shape[:-1], pw.n)
+    a_amax = x2.abs().amax().to(torch.float32)
+    a_gs = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax.clamp(min=1e-12)).reshape(1)
+    aq, a_sf = ops.scaled_fp4_quant(x2.to(torch.bfloat16), a_gs, True)
+    alpha = (1.0 / (a_gs * pw.global_scale)).to(torch.float32)
+    # The quant op returns swizzled scales packed 4-per-int32; the GEMM takes
+    # the same bytes typed as fp8-e4m3.
+    y = ops.cutlass_scaled_fp4_mm(aq, pw.qweight, _as_fp8(a_sf), _as_fp8(pw.sf), alpha)
+    return y.reshape(*x.shape[:-1], pw.n)
+def gemm_swiglu(pw: PackedWeight, x: torch.Tensor) -> torch.Tensor:
+    """silu(x @ W_gate.T) * (x @ W_up.T) for a SwiGLU-packed matrix."""
+    if not pw.swiglu:
+        raise ValueError("packed matrix was not packed with pack_swiglu")
+    y = gemm(pw, x)
+    half = pw.n // 2
+    return torch.nn.functional.silu(y[..., :half]) * y[..., half:]

build/torch-stable-abi29-cu130-x86_64-linux/layers.py ADDED Viewed

	@@ -0,0 +1,215 @@

+import torch
+import torch.nn as nn
+from .functional import gemm
+from ._ops import ops
+from ._pack import PackedWeight
+class NVFP4Linear(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(self, x):
+        pw = PackedWeight(
+            qweight=self.weight,
+            sf=self.weight_sf,
+            global_scale=self.weight_global_scale,
+            n=self.out_features,
+            k=self.in_features,
+            sf_rowmajor=getattr(self, "weight_sf_rowmajor", None),
+        )
+        return gemm(pw, x)
+def _ntokens(x):
+    return x.numel() // x.shape[-1]
+def _fused_gemv(module, x):
+    return ops.nvfp4_gemv(
+        x.reshape(-1, x.shape[-1]).to(torch.bfloat16),
+        module.nvfp4_fused_qweight,
+        module.nvfp4_fused_sf,
+        module.nvfp4_fused_alpha,
+    )
+class NVFP4FusedSwiGLUMLP(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(self, x):
+        if _ntokens(x) <= 2 and hasattr(self, "nvfp4_fused_qweight"):
+            x2 = x.reshape(-1, x.shape[-1]).to(torch.bfloat16)
+            h = ops.nvfp4_gemv_swiglu(
+                x2,
+                self.nvfp4_fused_qweight,
+                self.nvfp4_fused_sf,
+                self.nvfp4_fused_alpha,
+            )
+            return self.down_proj(h.reshape(*x.shape[:-1], -1))
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+class NVFP4GatedAttention(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(
+        self,
+        hidden_states,
+        position_embeddings=None,
+        attention_mask=None,
+        past_key_values=None,
+        **kwargs,
+    ):
+        from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+        og = None
+        if _ntokens(hidden_states) <= 2 and hasattr(self, "nvfp4_fused_qweight"):
+            fused = _fused_gemv(self, hidden_states)
+            q, k, v, *rest = torch.split(fused, self.nvfp4_fused_splits, dim=-1)
+            og = rest[0] if rest else None
+        else:
+            q = self.q_proj(hidden_states)
+            k = self.k_proj(hidden_states)
+            v = self.v_proj(hidden_states)
+        query_states = q.reshape(hidden_shape).transpose(1, 2)
+        key_states = k.reshape(hidden_shape).transpose(1, 2)
+        value_states = v.reshape(hidden_shape).transpose(1, 2)
+        norm = getattr(self, "qk_norm", None)
+        if norm is not None:
+            query_states = norm(query_states) * getattr(self, "qk_scale_factor", 1.0)
+            key_states = norm(key_states)
+        if position_embeddings is not None:
+            cos, sin = position_embeddings
+            query_states, key_states = self.nvfp4_rope(
+                query_states, key_states, cos, sin
+            )
+        if past_key_values is not None:
+            key_states, value_states = past_key_values.update(
+                key_states, value_states, self.layer_idx
+            )
+        attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, self.nvfp4_eager
+        )
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=self.attention_dropout if self.training else 0.0,
+            scaling=self.scaling,
+            sliding_window=getattr(self, "sliding_window", None),
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        if og is None and getattr(self, "gate_proj", None) is not None:
+            og = self.gate_proj(hidden_states)
+        o = self.o_proj
+        if og is not None and _ntokens(attn_output) <= 2 and _has_nvfp4(o):
+            a2 = attn_output.reshape(-1, attn_output.shape[-1])
+            og2 = og.reshape(a2.shape).contiguous()
+            alpha = (1.0 / o.weight_global_scale.float()).reshape(1)
+            y = ops.nvfp4_gemv_gated(a2, og2, o.weight, o.weight_sf_rowmajor, alpha)
+            return y.reshape(*input_shape, -1), attn_weights
+        if og is not None:
+            attn_output = attn_output * torch.sigmoid(og.reshape(attn_output.shape))
+        return o(attn_output), attn_weights
+def _has_nvfp4(m):
+    return m is not None and hasattr(m, "weight_sf_rowmajor")
+def _is_silu(act):
+    return isinstance(act, nn.SiLU) or type(act).__name__ == "SiLUActivation"
+def _attach_fused(module, mods):
+    # Keep a single packed allocation and expose row views back through the
+    # original projection modules.  Previously fusion allocated concatenated
+    # weights while retaining an independent copy in every projection.
+    # Attention and SwiGLU use row-concatenated storage; the latter is handled
+    # by the GEMV kernel using the gate/up half offset.
+    fused_qweight = torch.cat([m.weight for m in mods], 0).contiguous()
+    fused_sf = torch.cat([m.weight_sf_rowmajor for m in mods], 0).contiguous()
+    module.nvfp4_fused_qweight = fused_qweight
+    module.nvfp4_fused_sf = fused_sf
+    module.nvfp4_fused_alpha = torch.cat(
+        [
+            (1.0 / m.weight_global_scale.float())
+            .reshape(1)
+            .expand(m.out_features)
+            .contiguous()
+            for m in mods
+        ]
+    )
+    module.nvfp4_fused_splits = [m.out_features for m in mods]
+    offset = 0
+    for m in mods:
+        end = offset + m.out_features
+        m.weight = fused_qweight[offset:end]
+        m.weight_sf_rowmajor = fused_sf[offset:end]
+        offset = end
+def _capture_helpers(cls, rope_fn):
+    if getattr(cls, "nvfp4_eager", None) is not None:
+        return
+    fwd = cls.forward
+    names = ("apply_rotary_pos_emb_interleave", "apply_rotary_pos_emb")
+    rope = rope_fn or next(
+        (fwd.__globals__[n] for n in names if n in fwd.__code__.co_names), None
+    )
+    cls.nvfp4_rope = staticmethod(rope) if rope is not None else None
+    cls.nvfp4_eager = staticmethod(fwd.__globals__["eager_attention_forward"])
+def _mark(cls, layer_name):
+    if getattr(cls, "kernel_layer_name", None) is not None:
+        return
+    try:
+        from kernels import use_kernel_forward_from_hub
+        use_kernel_forward_from_hub(layer_name)(cls)
+    except Exception:
+        cls.kernel_layer_name = layer_name
+def fuse_decode_projections(model, rope_fn=None):
+    n = 0
+    for mod in model.modules():
+        qkv = [getattr(mod, p, None) for p in ("q_proj", "k_proj", "v_proj")]
+        if (
+            all(_has_nvfp4(p) for p in qkv)
+            and hasattr(mod, "o_proj")
+            and hasattr(mod, "layer_idx")
+        ):
+            gate = getattr(mod, "gate_proj", None)
+            _attach_fused(mod, qkv + ([gate] if _has_nvfp4(gate) else []))
+            _capture_helpers(type(mod), rope_fn)
+            _mark(type(mod), "NVFP4GatedAttention")
+            n += 1
+            continue
+        gu = [getattr(mod, p, None) for p in ("gate_proj", "up_proj")]
+        if (
+            all(_has_nvfp4(p) for p in gu)
+            and hasattr(mod, "down_proj")
+            and _is_silu(getattr(mod, "act_fn", None))
+            and gu[0].out_features == gu[1].out_features
+        ):
+            _attach_fused(mod, gu)
+            _mark(type(mod), "NVFP4SwiGLUMLP")
+            n += 1
+    return n

build/torch-stable-abi29-cu130-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "nvfp4-gemm",
+  "id": "_nvfp4_gemm_cuda_9c5a0e3",
+  "version": 1,
+  "license": "Apache-2.0",
+  "upstream": "git+https://github.com/vllm-project/vllm.git",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0a",
+      "12.0a"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "ARXik9k13ZmZ/1tDxHgH6bQyJ8Cn6wmCrz5WEM+DABI=",
+      "_nvfp4_gemm_cuda_9c5a0e3.abi3.so": "Z31NqhSDHLFzp2bNHACtYuM6z/1mUEaF2EFnmlPL0xU=",
+      "_ops.py": "+jBXp3Uoatvj6U0lcsYwN/3KdSd8+Aknhm5T+xbff6w=",
+      "_pack.py": "jATF2LhpSLUEiPWSrhsCWeyxIyn80CSy2QJNkKMADlc=",
+      "functional.py": "ZTd0hcUOkDdU9ZvpQMPC81KBP3FUvkmfZu7d7mhZRss=",
+      "layers.py": "asplwNUY3+drs57APjw3J9kFLtswK9YI0SxRVHZj2Hg=",
+      "nvfp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "9c5a0e35e9733ee93d458d2d4d5814f67d209d29",
+      "dirty": false
+    }
+  }
+}

build/torch-stable-abi29-cu130-x86_64-linux/metadata.json.sigstore ADDED Viewed

	@@ -0,0 +1 @@

+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHSzCCBtGgAwIBAgIUHAfPbhyAe3cC5esJqPyqePr8GwUwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYwODA2MTQwMzQ3WhcNMjYwODA2MTQxMzQ3WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEWUCsho2YzsyZePsqOe+UxlvRqdrcSrEw5hWkuhJezhApQK2pVybhQB/CKQDwkqHePS1yPoZMs7arsRTrDvp71KOCBfAwggXsMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUMXuHbek4tvcWcO8sCL27wc77BhwwHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wawYDVR0RAQH/BGEwX4ZdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDkGCisGAQQBg78wAQEEK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wHwYKKwYBBAGDvzABAgQRd29ya2Zsb3dfZGlzcGF0Y2gwNgYKKwYBBAGDvzABAwQoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTATBgorBgEEAYO/MAEEBAVCdWlsZDArBgorBgEEAYO/MAEFBB1odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTAdBgorBgEEAYO/MAEGBA9yZWZzL2hlYWRzL21haW4wOwYKKwYBBAGDvzABCAQtDCtodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMG0GCisGAQQBg78wAQkEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wAQoEKgwoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTAbBgorBgEEAYO/MAELBA0MC3NlbGYtaG9zdGVkMEAGCisGAQQBg78wAQwEMgwwaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MDgGCisGAQQBg78wAQ0EKgwoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTAfBgorBgEEAYO/MAEOBBEMD3JlZnMvaGVhZHMvbWFpbjAaBgorBgEEAYO/MAEPBAwMCjEwNzE0NzU1MjkwLgYKKwYBBAGDvzABEAQgDB5odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2UwGAYKKwYBBAGDvzABEQQKDAgyNTcyMDc0MzBtBgorBgEEAYO/MAESBF8MXWh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eS8uZ2l0aHViL3dvcmtmbG93cy9idWlsZC55YW1sQHJlZnMvaGVhZHMvbWFpbjA4BgorBgEEAYO/MAETBCoMKDljNWEwZTM1ZTk3MzNlZTkzZDQ1OGQyZDRkNTgxNGY2N2QyMDlkMjkwIQYKKwYBBAGDvzABFAQTDBF3b3JrZmxvd19kaXNwYXRjaDBkBgorBgEEAYO/MAEVBFYMVGh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eS9hY3Rpb25zL3J1bnMvMzExMDgyOTgwNDUvYXR0ZW1wdHMvMTAWBgorBgEEAYO/MAEWBAgMBnB1YmxpYzBGBgorBgEEAYO/MAEYBDgMNnJlcG86aHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHk6cmVmOnJlZnMvaGVhZHMvbWFpbjCBigYKKwYBBAHWeQIEAgR8BHoAeAB2AN09MGrGxxEyYxkeHJlnNwKiSl643jyt/4eKcoAvKe6OAAABn9djJjQAAAQDAEcwRQIhAIIWCYxW+rI6aaMi9HgD2DleYR6J+F6YqKXUxUv4Z4GmAiAyg9XAQCiB5lJQkeWNWmg/JPDvfOUMGlJVQFdOpjX/bzAKBggqhkjOPQQDAwNoADBlAjAjA7B7LOxYtc7NF049oP6Qo975a2A5zVyU/BWuCOQov+xcw5zDliXhCWXOLNW2BnACMQD0H11DuJC9QXwQNmHwj+A29g/LndNPkjV9AIyBq6hwTfWIEED6AkpjmuOSkbTrm2A="}, "tlogEntries":[{"logIndex":"2360822606", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1786025027", "inclusionPromise":{"signedEntryTimestamp":"MEQCIALdVIyCoXgz+SjRmxpJQDqgY8ecSWQymgLlX9xe2X61AiAUBt7xZWbAvhvQVPlt5yysMBXgs6LV+l2xYpfTXZ2TMA=="}, "inclusionProof":{"logIndex":"2238918344", "rootHash":"eN8oZuSEFGs70/BLfp19zZEf/Ef5N6oImTmtz+Jny7c=", "treeSize":"2238918358", "hashes":["OD35pd+GVFggfOcYGAvxIqbLmRi+yMYiisbcPtNJdJk=", "ohU4TYqXEjN0FOll1utEBukEQUc2a8glNqdj9lVxaqk=", "HErYWbjZJEkdOhwWC3iOORjGwhaO9z3YtuqIZsf55ek=", "qa31WzmDahZLK7TXE00fLX/3SyJLF2NCAmvEO941Ga4=", "NCmw55uH4clQcrqrPxLkqS6vGExt+NgyLt4oB68au6o=", "m06N3WcrfN7DqNJzhl3g+U0foWjztV58UZHFQYl3DGE=", "Fw0l2YFqeuLVAAuiRC2S6aMQd7o0x2qVKJK00oNkgcM=", "yOi1SZCAJyQeGFNfXEyJmZfgE/yHiViHJh7AkVStYx0=", "ZDDgBLJeZMIedbwsKMeIdE+rnhT6LFyxK01q4RQswB8=", "1QfEqQEF22LcM0N+z5i77k0Oy00Arz4yjX7D1LTJP34=", "tSch9ErXVKdLoCFPSZ8D8aLt8T+YHXh/0cV7rQ+HxWc=", "VN19Pd6HJSN2xUEEfXlAK+NBL7uztCXLEaUeuJxgAbY=", "2DzQwQqFGfdTX3BVYQkcDxiT/TBV7lnr+ya29yCNm+o=", "QVx27ou3XJyzOgzVsHRov+ie3M7wOfT492kpmkbAeBo=", "lLhES7her3Dzu/HSMpPVolTSfVQlKbD23EIsEwXq4V4=", "kVyfjHB6dSBvAqnJBVxvTpb/VPDK9zzq5/txahxnbhc=", "u23nWGlBSoQOMevXp72NpPaDjuN0qcB01i17Odcs3Wk=", "i5Zl8FZrDwxCDv2e2DNO2M8JvpR/c11ElvCZS53/teA=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n2238918358\neN8oZuSEFGs70/BLfp19zZEf/Ef5N6oImTmtz+Jny7c=\n\n— rekor.sigstore.dev wNI9ajBEAiBaLWisA7jij7yzI4AVOOST7ZFXt3aNtgN0/P2yM2mVmAIgAzBqsGjinhohmAEg/l+k00weJZykRapSeglzTmmpNBw=\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiI5ZGIxY2IxOGEzZGIzMWU5N2E1MjUzMmYzZjcxMzBkNWQxMTYxYWIzNjM3NmZjZTUwNGVlN2YxMDYwZWQxOWM4In19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FUUNJSHQ1eC83V0FzZWwyaWpqSXl5SGwyclgvUyt3U09jUkVwYUgwalRjeUpmUkFpQjl4SzRERlB4SFdkMjJRZHkyV0o2a3g2Z0lGZ3BpZ2NGczBZbVpGS1MwZFE9PSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaFRla05EUW5SSFowRjNTVUpCWjBsVlNFRm1VR0pvZVVGbE0yTkROV1Z6U25GUWVYRmxVSEk0UjNkVmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFpkMDlFUVRKTlZGRjNUWHBSTTFkb1kwNU5hbGwzVDBSQk1rMVVVWGhOZWxFelYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZYVlVOemFHOHlXWHB6ZVZwbFVITnhUMlVyVlhoc2RsSnhaSEpqVTNKRmR6Vm9WMnNLZFdoS1pYcG9RWEJSU3pKd1ZubGlhRkZDTDBOTFVVUjNhM0ZJWlZCVE1YbFFiMXBOY3pkaGNuTlNWSEpFZG5BM01VdFBRMEptUVhkbloxaHpUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXUkNaMDVXU0ZFMFJVWm5VVlZOV0hWSUNtSmxhelIwZG1OWFkwODRjME5NTWpkM1l6YzNRbWgzZDBoM1dVUldVakJxUWtKbmQwWnZRVlV6T1ZCd2VqRlphMFZhWWpWeFRtcHdTMFpYYVhocE5Ga0tXa1E0ZDJGM1dVUldVakJTUVZGSUwwSkhSWGRZTkZwa1lVaFNNR05JVFRaTWVUbHVZVmhTYjJSWFNYVlpNamwwVERKb01Wb3laSEJpYldSdFdWZE9iQXBNTW5Sc1kyMDFiR0pJVFhSWk1qbDBZbGhXZFdGWVVqVk1lVFZ1WVZoU2IyUlhTWFprTWpsNVlUSmFjMkl6WkhwTU1rb3hZVmQ0YTB4dWJHaGlWM2hCQ21OdFZtMWplVGx2V2xkR2EyTjVPWFJaVjJ4MVRVUnJSME5wYzBkQlVWRkNaemM0ZDBGUlJVVkxNbWd3WkVoQ2VrOXBPSFprUnpseVdsYzBkVmxYVGpBS1lWYzVkV041Tlc1aFdGSnZaRmRLTVdNeVZubFpNamwxWkVkV2RXUkROV3BpTWpCM1NIZFpTMHQzV1VKQ1FVZEVkbnBCUWtGblVWSmtNamw1WVRKYWN3cGlNMlJtV2tkc2VtTkhSakJaTW1kM1RtZFpTMHQzV1VKQ1FVZEVkbnBCUWtGM1VXOVBWMDB4V1ZSQ2JFMTZWbXhQVkdONlRUSldiRTlVVG10T1JGVTBDbHBFU210T1IxRXhUMFJGTUZwcVdUTmFSRWwzVDFkUmVVOVVRVlJDWjI5eVFtZEZSVUZaVHk5TlFVVkZRa0ZXUTJSWGJITmFSRUZ5UW1kdmNrSm5SVVVLUVZsUEwwMUJSVVpDUWpGdlpGZGtibUZYTlc1YWJVWnFXbE01Y2xwWVNuVmFWM2g2VEZkT2RtSlhNVEZpYld3d1pWUkJaRUpuYjNKQ1owVkZRVmxQTHdwTlFVVkhRa0U1ZVZwWFducE1NbWhzV1ZkU2Vrd3lNV2hoVnpSM1QzZFpTMHQzV1VKQ1FVZEVkbnBCUWtOQlVYUkVRM1J2WkVoU2QyTjZiM1pNTTFKMkNtRXlWblZNYlVacVpFZHNkbUp1VFhWYU1td3dZVWhXYVdSWVRteGpiVTUyWW01U2JHSnVVWFZaTWpsMFRVY3dSME5wYzBkQlVWRkNaemM0ZDBGUmEwVUtXSGQ0WkdGSVVqQmpTRTAyVEhrNWJtRllVbTlrVjBsMVdUSTVkRXd5YURGYU1tUndZbTFrYlZsWFRteE1NblJzWTIwMWJHSklUWFJaTWpsMFlsaFdkUXBoV0ZJMVRIazFibUZZVW05a1YwbDJaREk1ZVdFeVduTmlNMlI2VERKS01XRlhlR3RNYm14b1lsZDRRV050Vm0xamVUbHZXbGRHYTJONU9YUlpWMngxQ2sxRVowZERhWE5IUVZGUlFtYzNPSGRCVVc5RlMyZDNiMDlYVFRGWlZFSnNUWHBXYkU5VVkzcE5NbFpzVDFST2EwNUVWVFJhUkVwclRrZFJNVTlFUlRBS1dtcFpNMXBFU1hkUFYxRjVUMVJCWWtKbmIzSkNaMFZGUVZsUEwwMUJSVXhDUVRCTlF6Tk9iR0pIV1hSaFJ6bDZaRWRXYTAxRlFVZERhWE5IUVZGUlFncG5OemgzUVZGM1JVMW5kM2RoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlUxRVowZERhWE5IUVZGUlFtYzNPSGRCVVRCRlMyZDNiMDlYVFRGWlZFSnNUWHBXYkU5VVkzcE5NbFpzVDFST2EwNUVWVFFLV2tSS2EwNUhVVEZQUkVVd1dtcFpNMXBFU1hkUFYxRjVUMVJCWmtKbmIzSkNaMFZGUVZsUEwwMUJSVTlDUWtWTlJETktiRnB1VFhaaFIxWm9Xa2hOZGdwaVYwWndZbXBCWVVKbmIzSkNaMFZGUVZsUEwwMUJSVkJDUVhkTlEycEZkMDU2UlRCT2VsVXhUV3ByZDB4bldVdExkMWxDUWtGSFJIWjZRVUpGUVZGbkNrUkNOVzlrU0ZKM1kzcHZka3d5WkhCa1IyZ3hXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWM1IwRlpTMHQzV1VKQ1FVZEVkbnBCUWtWUlVVc0tSRUZuZVU1VVkzbE5SR013VFhwQ2RFSm5iM0pDWjBWRlFWbFBMMDFCUlZOQ1JqaE5XRmRvTUdSSVFucFBhVGgyV2pKc01HRklWbWxNYlU1MllsTTVid3BrVjJSdVlWYzFibHB0Um1wYVV6bHlXbGhLZFZwWGVIcE1WMDUyWWxjeE1XSnRiREJsVXpoMVdqSnNNR0ZJVm1sTU0yUjJZMjEwYldKSE9UTmplVGxwQ21SWGJITmFRelUxV1ZjeGMxRklTbXhhYmsxMllVZFdhRnBJVFhaaVYwWndZbXBCTkVKbmIzSkNaMFZGUVZsUEwwMUJSVlJDUTI5TlMwUnNhazVYUlhjS1dsUk5NVnBVYXpOTmVrNXNXbFJyZWxwRVVURlBSMUY1V2tSU2EwNVVaM2hPUjFreVRqSlJlVTFFYkd0TmFtdDNTVkZaUzB0M1dVSkNRVWRFZG5wQlFncEdRVkZVUkVKR00ySXpTbkphYlhoMlpERTVhMkZZVG5kWldGSnFZVVJDYTBKbmIzSkNaMFZGUVZsUEwwMUJSVlpDUmxsTlZrZG9NR1JJUW5wUGFUaDJDbG95YkRCaFNGWnBURzFPZG1KVE9XOWtWMlJ1WVZjMWJscHRSbXBhVXpseVdsaEtkVnBYZUhwTVYwNTJZbGN4TVdKdGJEQmxVemxvV1ROU2NHSXlOWG9LVEROS01XSnVUWFpOZWtWNFRVUm5lVTlVWjNkT1JGVjJXVmhTTUZwWE1YZGtTRTEyVFZSQlYwSm5iM0pDWjBWRlFWbFBMMDFCUlZkQ1FXZE5RbTVDTVFwWmJYaHdXWHBDUjBKbmIzSkNaMFZGUVZsUEwwMUJSVmxDUkdkTlRtNUtiR05IT0RaaFNGWnVXakpzZFZveVdtaFpNbFYyWVRKV2VXSnRWbk5qZVRGcUNtSXlNWFJrVnpWd1pFaHJObU50Vm0xUGJrcHNXbTVOZG1GSFZtaGFTRTEyWWxkR2NHSnFRMEpwWjFsTFMzZFpRa0pCU0ZkbFVVbEZRV2RTT0VKSWIwRUtaVUZDTWtGT01EbE5SM0pIZUhoRmVWbDRhMlZJU214dVRuZExhVk5zTmpRemFubDBMelJsUzJOdlFYWkxaVFpQUVVGQlFtNDVaR3BLYWxGQlFVRlJSQXBCUldOM1VsRkphRUZKU1ZkRFdYaFhLM0pKTm1GaFRXazVTR2RFTWtSc1pWbFNOa29yUmpaWmNVdFlWWGhWZGpSYU5FZHRRV2xCZVdjNVdFRlJRMmxDQ2pWc1NsRnJaVmRPVjIxbkwwcFFSSFptVDFWTlIyeEtWbEZHWkU5d2FsZ3ZZbnBCUzBKblozRm9hMnBQVUZGUlJFRjNUbTlCUkVKc1FXcEJha0UzUWpjS1RFOTRXWFJqTjA1R01EUTViMUEyVVc4NU56VmhNa0UxZWxaNVZTOUNWM1ZEVDFGdmRpdDRZM2MxZWtSc2FWaG9RMWRZVDB4T1Z6SkNia0ZEVFZGRU1BcElNVEZFZFVwRE9WRllkMUZPYlVoM2FpdEJNamxuTDB4dVpFNVFhMnBXT1VGSmVVSnhObWgzVkdaWFNVVkZSRFpCYTNCcWJYVlBVMnRpVkhKdE1rRTlDaTB0TFMwdFJVNUVJRU5GVWxSSlJrbERRVlJGTFMwdExTMEsifX19fQ=="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyzADAgEAMIICwgYJKoZIhvcNAQcCoIICszCCAq8CAQMxDTALBglghkgBZQMEAgEwgbgGCyqGSIb3DQEJEAEEoIGoBIGlMIGiAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQghTN3Etc/xMW2ECEnnJ77fF4BFTlIInKJA+klU1KR364CFQDI1qpdKNJ+eB43rDLpCBw+sGN+HxgPMjAyNjA4MDYxNDAzNDdaMAMCAQGgMqQwMC4xFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEVMBMGA1UEAxMMc2lnc3RvcmUtdHNhoAAxggHcMIIB2AIBATBRMDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAsGCWCGSAFlAwQCAaCB/DAaBgkqhkiG9w0BCQMxDQYLKoZIhvcNAQkQAQQwHAYJKoZIhvcNAQkFMQ8XDTI2MDgwNjE0MDM0N1owLwYJKoZIhvcNAQkEMSIEIMpfp637Nc+LAhkPFU6iox6d4QDZrv1y+2irknjjsgXOMIGOBgsqhkiG9w0BCRACLzF/MH0wezB5BCCF+Se8B6tiysO0Q1bBDvyBssaIP9p6uebYcNnROs0FtzBVMD2kOzA5MRUwEwYDVQQKEwxzaWdzdG9yZS5kZXYxIDAeBgNVBAMTF3NpZ3N0b3JlLXRzYS1zZWxmc2lnbmVkAhQ6E1QvDJBh7rzBQy/Lio6LKiOLDDAKBggqhkjOPQQDAgRoMGYCMQC542sKD2Td6CYDzDxI/dgV+8SVjUZVYPo4mJTdgP6Hd7R+QAV73XhuCT3ZHaX+dO8CMQDrfFq1+W81IP5G6WQG9G5xqXqyDbIUsSmlZJHMsPkiK4+y30rW0rYtVZk8XTacep4="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"nbHLGKPbMel6UlMvP3Ew1dEWGrNjdvzlBO5/EGDtGcg="}, "signature":"MEQCIHt5x/7WAsel2ijjIyyHl2rX/S+wSOcREpaH0jTcyJfRAiB9xK4DFPxHWd22Qdy2WJ6kx6gIFgpigcFs0YmZFKS0dQ=="}}

build/torch-stable-abi29-cu130-x86_64-linux/nvfp4_gemm/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))

build/torch-stable-abi29-cu132-x86_64-linux/__init__.py ADDED Viewed

	@@ -0,0 +1,24 @@

+from . import layers
+from .layers import fuse_decode_projections
+from .functional import gemm, gemm_swiglu
+from ._pack import (
+    PackedWeight,
+    global_scale_for,
+    pack,
+    pack_swiglu,
+    quantize_reference,
+    swizzled_sf_shape,
+)
+__all__ = [
+    "PackedWeight",
+    "pack",
+    "pack_swiglu",
+    "quantize_reference",
+    "global_scale_for",
+    "swizzled_sf_shape",
+    "gemm",
+    "gemm_swiglu",
+    "layers",
+    "fuse_decode_projections",
+]

build/torch-stable-abi29-cu132-x86_64-linux/_nvfp4_gemm_cuda_9c5a0e3.abi3.so ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d789bb047d295c5147d5865809881daf3aa4384e30addbc61d5b2f4e03b23c4e
+size 3260632

build/torch-stable-abi29-cu132-x86_64-linux/_ops.py ADDED Viewed

	@@ -0,0 +1,9 @@

+import torch
+from . import _nvfp4_gemm_cuda_9c5a0e3
+ops = torch.ops._nvfp4_gemm_cuda_9c5a0e3
+def add_op_namespace_prefix(op_name: str):
+    """
+    Prefix op by namespace.
+    """
+    return f"_nvfp4_gemm_cuda_9c5a0e3::{op_name}"

build/torch-stable-abi29-cu132-x86_64-linux/_pack.py ADDED Viewed

	@@ -0,0 +1,110 @@

+from dataclasses import dataclass
+import torch
+from ._ops import ops
+FLOAT8_E4M3_MAX = 448.0
+FLOAT4_E2M1_MAX = 6.0
+SF_VEC_SIZE = 16  # elements sharing one FP8 scale
+def _round_up(x: int, m: int) -> int:
+    return (x + m - 1) // m * m
+def swizzled_sf_shape(rows: int, cols: int) -> tuple[int, int]:
+    """Shape of the swizzled scale-factor tensor (dtype int32) produced by
+    `scaled_fp4_quant(..., is_sf_swizzled_layout=True)` for a [rows, cols]
+    input. Mirrors `computeSwizzledSFShape` in nvfp4_utils.cuh: 128-row tiles,
+    4 FP8 scales packed per int32 column."""
+    return _round_up(rows, 128), _round_up(cols // SF_VEC_SIZE, 4) // 4
+def global_scale_for(w: torch.Tensor) -> torch.Tensor:
+    """Per-tensor second-level scale: FP8_MAX * FP4_MAX / amax."""
+    amax = w.abs().amax().to(torch.float32)
+    return (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / amax.clamp(min=1e-12)).reshape(1)
+@dataclass
+class PackedWeight:
+    """NVFP4 representation of a bf16 matrix [n, k]."""
+    qweight: torch.Tensor  # uint8 [n, k/2], two e2m1 values per byte
+    sf: torch.Tensor  # int32, swizzled FP8-E4M3 block scales
+    global_scale: torch.Tensor  # fp32 [1]
+    n: int
+    k: int
+    swiglu: bool = False
+    # uint8 [n, k/16] row-major FP8-E4M3 block scales, consumed by the W4A16
+    # GEMV decode path (None when only the swizzled layout is available).
+    sf_rowmajor: torch.Tensor | None = None
+    def to(self, device) -> "PackedWeight":
+        return PackedWeight(
+            qweight=self.qweight.to(device),
+            sf=self.sf.to(device),
+            global_scale=self.global_scale.to(device),
+            n=self.n,
+            k=self.k,
+            swiglu=self.swiglu,
+            sf_rowmajor=(
+                None if self.sf_rowmajor is None else self.sf_rowmajor.to(device)
+            ),
+        )
+def pack(
+    w: torch.Tensor, device: str | torch.device = "cuda", swiglu: bool = False
+) -> PackedWeight:
+    """Quantize a bf16/fp16 matrix [n, k] to NVFP4 (requires sm100+)."""
+    if w.dim() != 2 or not w.is_floating_point():
+        raise ValueError("pack() expects a 2-D floating-point weight")
+    n, k = w.shape
+    if k % SF_VEC_SIZE != 0:
+        raise ValueError(f"in_features must be a multiple of {SF_VEC_SIZE}, got {k}")
+    wd = w.detach().to(device, torch.bfloat16).contiguous()
+    gs = global_scale_for(wd).to(device)
+    qweight, sf = ops.scaled_fp4_quant(wd, gs, True)
+    # Row-major scales for the W4A16 GEMV decode path; the qweight output is
+    # identical to the swizzled call's, so it is discarded.
+    _, sf_rowmajor = ops.scaled_fp4_quant(wd, gs, False)
+    return PackedWeight(
+        qweight=qweight,
+        sf=sf,
+        global_scale=gs,
+        n=n,
+        k=k,
+        swiglu=swiglu,
+        sf_rowmajor=sf_rowmajor,
+    )
+def pack_swiglu(
+    w_gate: torch.Tensor, w_up: torch.Tensor, device: str | torch.device = "cuda"
+) -> PackedWeight:
+    """Pack gate and up projections [n, k] as one [2n, k] matrix; the GEMM
+    output has gate values in columns [0, n) and up in [n, 2n)."""
+    if w_gate.shape != w_up.shape:
+        raise ValueError("gate and up matrices must have the same shape")
+    return pack(torch.cat([w_gate, w_up], dim=0), device=device, swiglu=True)
+def quantize_reference(w: torch.Tensor) -> torch.Tensor:
+    """Pure-torch NVFP4 emulation (no GPU kernel): quantize + dequantize to
+    the same numerics the kernel produces, for tests and quality analysis."""
+    orig_dtype = w.dtype
+    wf = w.float()
+    gs = global_scale_for(wf)  # fp8_max*fp4_max/amax
+    n, k = wf.shape
+    g = wf.reshape(n, k // SF_VEC_SIZE, SF_VEC_SIZE)
+    # Block scale: amax/FP4_MAX * global_scale, stored in FP8-E4M3.
+    block_amax = g.abs().amax(-1, keepdim=True)
+    sf = (block_amax / FLOAT4_E2M1_MAX) * gs
+    sf = sf.to(torch.float8_e4m3fn).float().clamp(min=1e-12)
+    scaled = g * (gs / sf)
+    # e2m1 grid: {0, .5, 1, 1.5, 2, 3, 4, 6} with signs.
+    grid = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
+    q = grid[(scaled.abs().unsqueeze(-1) - grid).abs().argmin(-1)] * scaled.sign()
+    return (q * sf / gs).reshape(n, k).to(orig_dtype)

build/torch-stable-abi29-cu132-x86_64-linux/functional.py ADDED Viewed

	@@ -0,0 +1,75 @@

+import torch
+from ._ops import add_op_namespace_prefix, ops
+from ._pack import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX, PackedWeight, swizzled_sf_shape
+@torch.library.register_fake(add_op_namespace_prefix("scaled_fp4_quant"))
+def _quant_fake(input, input_global_scale, is_sf_swizzled_layout):
+    n = input.size(-1)
+    m = input.numel() // n
+    out = input.new_empty((m, n // 2), dtype=torch.uint8)
+    if is_sf_swizzled_layout:
+        sf_m, sf_n = swizzled_sf_shape(m, n)
+        sf = input.new_empty((sf_m, sf_n), dtype=torch.int32)
+    else:
+        sf = input.new_empty((m, n // 16), dtype=torch.uint8)
+    return out, sf
+@torch.library.register_fake(add_op_namespace_prefix("cutlass_scaled_fp4_mm"))
+def _mm_fake(a, b, a_sf, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv"))
+def _gemv_fake(a, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv_swiglu"))
+def _gemv_swiglu_fake(a, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0) // 2), dtype=torch.bfloat16)
+@torch.library.register_fake(add_op_namespace_prefix("nvfp4_gemv_gated"))
+def _gemv_gated_fake(a, g, b, b_sf, alpha):
+    return a.new_empty((a.size(0), b.size(0)), dtype=torch.bfloat16)
+def _as_fp8(sf: torch.Tensor) -> torch.Tensor:
+    return sf.view(torch.float8_e4m3fn) if sf.dtype == torch.int32 else sf
+def gemm(pw: PackedWeight, x: torch.Tensor) -> torch.Tensor:
+    """y[..., n] = x[..., k] @ W.T with dynamic NVFP4 activation quantization
+    (W4A4 on the Blackwell block-scaled tensor cores)."""
+    if x.shape[-1] != pw.k:
+        raise ValueError(f"expected {pw.k} input features, got {x.shape[-1]}")
+    x2 = x.reshape(-1, pw.k)
+    if not x2.is_contiguous():
+        x2 = x2.contiguous()
+    # Decode path: W4A16 GEMV keeps activations in bf16 (no activation
+    # quantization). The kernel re-reads the weight per output row, so it is
+    # only worthwhile for tiny m; prefill stays on the W4A4 CUTLASS path.
+    if x2.size(0) <= 2 and getattr(pw, "sf_rowmajor", None) is not None:
+        alpha = (1.0 / pw.global_scale).to(torch.float32).reshape(1)
+        y = ops.nvfp4_gemv(x2.to(torch.bfloat16), pw.qweight, pw.sf_rowmajor, alpha)
+        return y.reshape(*x.shape[:-1], pw.n)
+    a_amax = x2.abs().amax().to(torch.float32)
+    a_gs = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / a_amax.clamp(min=1e-12)).reshape(1)
+    aq, a_sf = ops.scaled_fp4_quant(x2.to(torch.bfloat16), a_gs, True)
+    alpha = (1.0 / (a_gs * pw.global_scale)).to(torch.float32)
+    # The quant op returns swizzled scales packed 4-per-int32; the GEMM takes
+    # the same bytes typed as fp8-e4m3.
+    y = ops.cutlass_scaled_fp4_mm(aq, pw.qweight, _as_fp8(a_sf), _as_fp8(pw.sf), alpha)
+    return y.reshape(*x.shape[:-1], pw.n)
+def gemm_swiglu(pw: PackedWeight, x: torch.Tensor) -> torch.Tensor:
+    """silu(x @ W_gate.T) * (x @ W_up.T) for a SwiGLU-packed matrix."""
+    if not pw.swiglu:
+        raise ValueError("packed matrix was not packed with pack_swiglu")
+    y = gemm(pw, x)
+    half = pw.n // 2
+    return torch.nn.functional.silu(y[..., :half]) * y[..., half:]

build/torch-stable-abi29-cu132-x86_64-linux/layers.py ADDED Viewed

	@@ -0,0 +1,215 @@

+import torch
+import torch.nn as nn
+from .functional import gemm
+from ._ops import ops
+from ._pack import PackedWeight
+class NVFP4Linear(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(self, x):
+        pw = PackedWeight(
+            qweight=self.weight,
+            sf=self.weight_sf,
+            global_scale=self.weight_global_scale,
+            n=self.out_features,
+            k=self.in_features,
+            sf_rowmajor=getattr(self, "weight_sf_rowmajor", None),
+        )
+        return gemm(pw, x)
+def _ntokens(x):
+    return x.numel() // x.shape[-1]
+def _fused_gemv(module, x):
+    return ops.nvfp4_gemv(
+        x.reshape(-1, x.shape[-1]).to(torch.bfloat16),
+        module.nvfp4_fused_qweight,
+        module.nvfp4_fused_sf,
+        module.nvfp4_fused_alpha,
+    )
+class NVFP4FusedSwiGLUMLP(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(self, x):
+        if _ntokens(x) <= 2 and hasattr(self, "nvfp4_fused_qweight"):
+            x2 = x.reshape(-1, x.shape[-1]).to(torch.bfloat16)
+            h = ops.nvfp4_gemv_swiglu(
+                x2,
+                self.nvfp4_fused_qweight,
+                self.nvfp4_fused_sf,
+                self.nvfp4_fused_alpha,
+            )
+            return self.down_proj(h.reshape(*x.shape[:-1], -1))
+        return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+class NVFP4GatedAttention(nn.Module):
+    can_torch_compile = True
+    has_backward = False
+    def forward(
+        self,
+        hidden_states,
+        position_embeddings=None,
+        attention_mask=None,
+        past_key_values=None,
+        **kwargs,
+    ):
+        from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+        og = None
+        if _ntokens(hidden_states) <= 2 and hasattr(self, "nvfp4_fused_qweight"):
+            fused = _fused_gemv(self, hidden_states)
+            q, k, v, *rest = torch.split(fused, self.nvfp4_fused_splits, dim=-1)
+            og = rest[0] if rest else None
+        else:
+            q = self.q_proj(hidden_states)
+            k = self.k_proj(hidden_states)
+            v = self.v_proj(hidden_states)
+        query_states = q.reshape(hidden_shape).transpose(1, 2)
+        key_states = k.reshape(hidden_shape).transpose(1, 2)
+        value_states = v.reshape(hidden_shape).transpose(1, 2)
+        norm = getattr(self, "qk_norm", None)
+        if norm is not None:
+            query_states = norm(query_states) * getattr(self, "qk_scale_factor", 1.0)
+            key_states = norm(key_states)
+        if position_embeddings is not None:
+            cos, sin = position_embeddings
+            query_states, key_states = self.nvfp4_rope(
+                query_states, key_states, cos, sin
+            )
+        if past_key_values is not None:
+            key_states, value_states = past_key_values.update(
+                key_states, value_states, self.layer_idx
+            )
+        attention_interface = ALL_ATTENTION_FUNCTIONS.get_interface(
+            self.config._attn_implementation, self.nvfp4_eager
+        )
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=self.attention_dropout if self.training else 0.0,
+            scaling=self.scaling,
+            sliding_window=getattr(self, "sliding_window", None),
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        if og is None and getattr(self, "gate_proj", None) is not None:
+            og = self.gate_proj(hidden_states)
+        o = self.o_proj
+        if og is not None and _ntokens(attn_output) <= 2 and _has_nvfp4(o):
+            a2 = attn_output.reshape(-1, attn_output.shape[-1])
+            og2 = og.reshape(a2.shape).contiguous()
+            alpha = (1.0 / o.weight_global_scale.float()).reshape(1)
+            y = ops.nvfp4_gemv_gated(a2, og2, o.weight, o.weight_sf_rowmajor, alpha)
+            return y.reshape(*input_shape, -1), attn_weights
+        if og is not None:
+            attn_output = attn_output * torch.sigmoid(og.reshape(attn_output.shape))
+        return o(attn_output), attn_weights
+def _has_nvfp4(m):
+    return m is not None and hasattr(m, "weight_sf_rowmajor")
+def _is_silu(act):
+    return isinstance(act, nn.SiLU) or type(act).__name__ == "SiLUActivation"
+def _attach_fused(module, mods):
+    # Keep a single packed allocation and expose row views back through the
+    # original projection modules.  Previously fusion allocated concatenated
+    # weights while retaining an independent copy in every projection.
+    # Attention and SwiGLU use row-concatenated storage; the latter is handled
+    # by the GEMV kernel using the gate/up half offset.
+    fused_qweight = torch.cat([m.weight for m in mods], 0).contiguous()
+    fused_sf = torch.cat([m.weight_sf_rowmajor for m in mods], 0).contiguous()
+    module.nvfp4_fused_qweight = fused_qweight
+    module.nvfp4_fused_sf = fused_sf
+    module.nvfp4_fused_alpha = torch.cat(
+        [
+            (1.0 / m.weight_global_scale.float())
+            .reshape(1)
+            .expand(m.out_features)
+            .contiguous()
+            for m in mods
+        ]
+    )
+    module.nvfp4_fused_splits = [m.out_features for m in mods]
+    offset = 0
+    for m in mods:
+        end = offset + m.out_features
+        m.weight = fused_qweight[offset:end]
+        m.weight_sf_rowmajor = fused_sf[offset:end]
+        offset = end
+def _capture_helpers(cls, rope_fn):
+    if getattr(cls, "nvfp4_eager", None) is not None:
+        return
+    fwd = cls.forward
+    names = ("apply_rotary_pos_emb_interleave", "apply_rotary_pos_emb")
+    rope = rope_fn or next(
+        (fwd.__globals__[n] for n in names if n in fwd.__code__.co_names), None
+    )
+    cls.nvfp4_rope = staticmethod(rope) if rope is not None else None
+    cls.nvfp4_eager = staticmethod(fwd.__globals__["eager_attention_forward"])
+def _mark(cls, layer_name):
+    if getattr(cls, "kernel_layer_name", None) is not None:
+        return
+    try:
+        from kernels import use_kernel_forward_from_hub
+        use_kernel_forward_from_hub(layer_name)(cls)
+    except Exception:
+        cls.kernel_layer_name = layer_name
+def fuse_decode_projections(model, rope_fn=None):
+    n = 0
+    for mod in model.modules():
+        qkv = [getattr(mod, p, None) for p in ("q_proj", "k_proj", "v_proj")]
+        if (
+            all(_has_nvfp4(p) for p in qkv)
+            and hasattr(mod, "o_proj")
+            and hasattr(mod, "layer_idx")
+        ):
+            gate = getattr(mod, "gate_proj", None)
+            _attach_fused(mod, qkv + ([gate] if _has_nvfp4(gate) else []))
+            _capture_helpers(type(mod), rope_fn)
+            _mark(type(mod), "NVFP4GatedAttention")
+            n += 1
+            continue
+        gu = [getattr(mod, p, None) for p in ("gate_proj", "up_proj")]
+        if (
+            all(_has_nvfp4(p) for p in gu)
+            and hasattr(mod, "down_proj")
+            and _is_silu(getattr(mod, "act_fn", None))
+            and gu[0].out_features == gu[1].out_features
+        ):
+            _attach_fused(mod, gu)
+            _mark(type(mod), "NVFP4SwiGLUMLP")
+            n += 1
+    return n

build/torch-stable-abi29-cu132-x86_64-linux/metadata.json ADDED Viewed

	@@ -0,0 +1,38 @@

+{
+  "name": "nvfp4-gemm",
+  "id": "_nvfp4_gemm_cuda_9c5a0e3",
+  "version": 1,
+  "license": "Apache-2.0",
+  "upstream": "git+https://github.com/vllm-project/vllm.git",
+  "python-depends": [],
+  "backend": {
+    "type": "cuda",
+    "archs": [
+      "10.0a",
+      "12.0a"
+    ]
+  },
+  "digest": {
+    "algorithm": "sha256",
+    "files": {
+      "__init__.py": "ARXik9k13ZmZ/1tDxHgH6bQyJ8Cn6wmCrz5WEM+DABI=",
+      "_nvfp4_gemm_cuda_9c5a0e3.abi3.so": "14m7BH0pXFFH1YZYCYgdrzqkOE4wrdvGHVsvTgOyPE4=",
+      "_ops.py": "+jBXp3Uoatvj6U0lcsYwN/3KdSd8+Aknhm5T+xbff6w=",
+      "_pack.py": "jATF2LhpSLUEiPWSrhsCWeyxIyn80CSy2QJNkKMADlc=",
+      "functional.py": "ZTd0hcUOkDdU9ZvpQMPC81KBP3FUvkmfZu7d7mhZRss=",
+      "layers.py": "asplwNUY3+drs57APjw3J9kFLtswK9YI0SxRVHZj2Hg=",
+      "nvfp4_gemm/__init__.py": "DFYPlrhXwYjEqCl/8n0SmWGZV8NFml5DPhMjKfv98GY="
+    }
+  },
+  "provenance": {
+    "kernel-builder": {
+      "version": "0.17.0-dev0",
+      "sha": "19aaa6421e674e9fecc352bbae6eab81d19a6bf4",
+      "dirty": false
+    },
+    "kernel": {
+      "sha": "9c5a0e35e9733ee93d458d2d4d5814f67d209d29",
+      "dirty": false
+    }
+  }
+}

build/torch-stable-abi29-cu132-x86_64-linux/metadata.json.sigstore ADDED Viewed

	@@ -0,0 +1 @@

+ {"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json", "verificationMaterial":{"certificate":{"rawBytes":"MIIHSzCCBtGgAwIBAgIUQP0jgzq6iUNWh+2Y8Rxg9VqrtCMwCgYIKoZIzj0EAwMwNzEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MR4wHAYDVQQDExVzaWdzdG9yZS1pbnRlcm1lZGlhdGUwHhcNMjYwODA2MTQwMzQ3WhcNMjYwODA2MTQxMzQ3WjAAMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEbvXQv6+bD792eCLro0ZEBoUesPS3o66yHG0PtgJa1NhBoB4yV2geGnvcXQLZq7PWvs5pl1I6ycA6O+pv3a0jKaOCBfAwggXsMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAdBgNVHQ4EFgQUcuoodiclwg80Xgmz2zHcxEaSRcMwHwYDVR0jBBgwFoAU39Ppz1YkEZb5qNjpKFWixi4YZD8wawYDVR0RAQH/BGEwX4ZdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDkGCisGAQQBg78wAQEEK2h0dHBzOi8vdG9rZW4uYWN0aW9ucy5naXRodWJ1c2VyY29udGVudC5jb20wHwYKKwYBBAGDvzABAgQRd29ya2Zsb3dfZGlzcGF0Y2gwNgYKKwYBBAGDvzABAwQoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTATBgorBgEEAYO/MAEEBAVCdWlsZDArBgorBgEEAYO/MAEFBB1odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eTAdBgorBgEEAYO/MAEGBA9yZWZzL2hlYWRzL21haW4wOwYKKwYBBAGDvzABCAQtDCtodHRwczovL3Rva2VuLmFjdGlvbnMuZ2l0aHVidXNlcmNvbnRlbnQuY29tMG0GCisGAQQBg78wAQkEXwxdaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5Ly5naXRodWIvd29ya2Zsb3dzL2J1aWxkLnlhbWxAcmVmcy9oZWFkcy9tYWluMDgGCisGAQQBg78wAQoEKgwoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTAbBgorBgEEAYO/MAELBA0MC3NlbGYtaG9zdGVkMEAGCisGAQQBg78wAQwEMgwwaHR0cHM6Ly9naXRodWIuY29tL2h1Z2dpbmdmYWNlL2tlcm5lbHMtY29tbXVuaXR5MDgGCisGAQQBg78wAQ0EKgwoOWM1YTBlMzVlOTczM2VlOTNkNDU4ZDJkNGQ1ODE0ZjY3ZDIwOWQyOTAfBgorBgEEAYO/MAEOBBEMD3JlZnMvaGVhZHMvbWFpbjAaBgorBgEEAYO/MAEPBAwMCjEwNzE0NzU1MjkwLgYKKwYBBAGDvzABEAQgDB5odHRwczovL2dpdGh1Yi5jb20vaHVnZ2luZ2ZhY2UwGAYKKwYBBAGDvzABEQQKDAgyNTcyMDc0MzBtBgorBgEEAYO/MAESBF8MXWh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eS8uZ2l0aHViL3dvcmtmbG93cy9idWlsZC55YW1sQHJlZnMvaGVhZHMvbWFpbjA4BgorBgEEAYO/MAETBCoMKDljNWEwZTM1ZTk3MzNlZTkzZDQ1OGQyZDRkNTgxNGY2N2QyMDlkMjkwIQYKKwYBBAGDvzABFAQTDBF3b3JrZmxvd19kaXNwYXRjaDBkBgorBgEEAYO/MAEVBFYMVGh0dHBzOi8vZ2l0aHViLmNvbS9odWdnaW5nZmFjZS9rZXJuZWxzLWNvbW11bml0eS9hY3Rpb25zL3J1bnMvMzExMDgyOTgwNDUvYXR0ZW1wdHMvMTAWBgorBgEEAYO/MAEWBAgMBnB1YmxpYzBGBgorBgEEAYO/MAEYBDgMNnJlcG86aHVnZ2luZ2ZhY2Uva2VybmVscy1jb21tdW5pdHk6cmVmOnJlZnMvaGVhZHMvbWFpbjCBigYKKwYBBAHWeQIEAgR8BHoAeAB2AN09MGrGxxEyYxkeHJlnNwKiSl643jyt/4eKcoAvKe6OAAABn9djKbEAAAQDAEcwRQIhAOsiNNSfv3dX77qcN9iuNZj2oTpkyRL3wAyNHTGKxZe0AiBP1JX+m9rtjPbYEXSF8Cw+hk5BrZ5fxzNiZC3TI9FfgjAKBggqhkjOPQQDAwNoADBlAjEA/s95wwFRDQUOX06PKVEt45K6RnhS+AUm70/LIXqbqE+4kFjW5ruFyV+potMxLYKYAjBaoT33gaX/2cnP/Es5mBs7UBabUm/CfFYL+CBf0OzFKMXsYn3Gb+VRfAghuXxY+qI="}, "tlogEntries":[{"logIndex":"2360822794", "logId":{"keyId":"wNI9atQGlz+VWfO6LRygH4QUfY/8W4RFwiT5i5WRgB0="}, "kindVersion":{"kind":"hashedrekord", "version":"0.0.1"}, "integratedTime":"1786025028", "inclusionPromise":{"signedEntryTimestamp":"MEQCIAJ+mz7Vh4YuC2Ju2kJBDk/nOMGzpY13mQRZEPtOVyB7AiBjx8QlP9qfjGWQWmDYA/QtrrVlnlcIWuGfMXeOK4yAyA=="}, "inclusionProof":{"logIndex":"2238918532", "rootHash":"rKIuTXmGd55gfj4rnFlrOuOrqWBpfOkbhZyhvBNAvj0=", "treeSize":"2238918554", "hashes":["l1vm1EL7VhYR/fL9IVTYJcPVc+efJzqct14GrWlyXLg=", "0F/8qEd4yf19WRRy8re+mSmkmNm9rrWnZOhzgKScN2w=", "Jz+PRA4FGGCnAdj3jIZjiHxchfAsf0y3ZBZdscAIVkw=", "DwgeNpdx/Xl0foZN932HJNaNZGxS/ApVyicyhFgh/3A=", "Khbu+Fuzutx3ADkt7AeAP4aiISNWAcz/kswjB45B21c=", "5WcoMXSpzvnq4KWL+iZGwWkjGDYd22yUN5/2n1rJbDw=", "yqmsZOsG1fWCdReSoyBV2DCsoA3XKvTEPnOnexLwYnI=", "yOi1SZCAJyQeGFNfXEyJmZfgE/yHiViHJh7AkVStYx0=", "ZDDgBLJeZMIedbwsKMeIdE+rnhT6LFyxK01q4RQswB8=", "1QfEqQEF22LcM0N+z5i77k0Oy00Arz4yjX7D1LTJP34=", "tSch9ErXVKdLoCFPSZ8D8aLt8T+YHXh/0cV7rQ+HxWc=", "VN19Pd6HJSN2xUEEfXlAK+NBL7uztCXLEaUeuJxgAbY=", "2DzQwQqFGfdTX3BVYQkcDxiT/TBV7lnr+ya29yCNm+o=", "QVx27ou3XJyzOgzVsHRov+ie3M7wOfT492kpmkbAeBo=", "lLhES7her3Dzu/HSMpPVolTSfVQlKbD23EIsEwXq4V4=", "kVyfjHB6dSBvAqnJBVxvTpb/VPDK9zzq5/txahxnbhc=", "u23nWGlBSoQOMevXp72NpPaDjuN0qcB01i17Odcs3Wk=", "i5Zl8FZrDwxCDv2e2DNO2M8JvpR/c11ElvCZS53/teA=", "xH/DCseLHr9eKoYT8qsORZK7zVdEGYWHuVtsVrD95wY="], "checkpoint":{"envelope":"rekor.sigstore.dev - 1193050959916656506\n2238918554\nrKIuTXmGd55gfj4rnFlrOuOrqWBpfOkbhZyhvBNAvj0=\n\n— rekor.sigstore.dev wNI9ajBFAiEAmIz2SLnVdzVNEMYAlAzbQ2Qqy+ULNVITLFWfsOxRQvYCIE9KfpPwYh1cVJZGK92tvhqJWkRphkx2o0JBP8+RXV8g\n"}}, "canonicalizedBody":"eyJhcGlWZXJzaW9uIjoiMC4wLjEiLCJraW5kIjoiaGFzaGVkcmVrb3JkIiwic3BlYyI6eyJkYXRhIjp7Imhhc2giOnsiYWxnb3JpdGhtIjoic2hhMjU2IiwidmFsdWUiOiIxNGUxZTJjODY3YzAyZDVlNGYyYTNjMmM2NjUyNzk3MTY1ZGJmMzlhODZmYTkwYzliNmQwOWZjYmQ1ZWRlZWVjIn19LCJzaWduYXR1cmUiOnsiY29udGVudCI6Ik1FWUNJUUMrSVJ0ZituN080NFVLMkJONlkybk4xN2hnaUNwUHpIRm9taklEL0p0K1NnSWhBT3plcTBFRk1WRGJEYisvVFViT3pUQWdRWFhCQS8vUURYS2tVbDVBeENDOSIsInB1YmxpY0tleSI6eyJjb250ZW50IjoiTFMwdExTMUNSVWRKVGlCRFJWSlVTVVpKUTBGVVJTMHRMUzB0Q2sxSlNVaFRla05EUW5SSFowRjNTVUpCWjBsVlVWQXdhbWQ2Y1RacFZVNVhhQ3N5V1RoU2VHYzVWbkZ5ZEVOTmQwTm5XVWxMYjFwSmVtb3dSVUYzVFhjS1RucEZWazFDVFVkQk1WVkZRMmhOVFdNeWJHNWpNMUoyWTIxVmRWcEhWakpOVWpSM1NFRlpSRlpSVVVSRmVGWjZZVmRrZW1SSE9YbGFVekZ3WW01U2JBcGpiVEZzV2tkc2FHUkhWWGRJYUdOT1RXcFpkMDlFUVRKTlZGRjNUWHBSTTFkb1kwNU5hbGwzVDBSQk1rMVVVWGhOZWxFelYycEJRVTFHYTNkRmQxbElDa3R2V2tsNmFqQkRRVkZaU1V0dldrbDZhakJFUVZGalJGRm5RVVZpZGxoUmRqWXJZa1EzT1RKbFEweHliekJhUlVKdlZXVnpVRk16YnpZMmVVaEhNRkFLZEdkS1lURk9hRUp2UWpSNVZqSm5aVWR1ZG1OWVVVeGFjVGRRVjNaek5YQnNNVWsyZVdOQk5rOHJjSFl6WVRCcVMyRlBRMEptUVhkbloxaHpUVUUwUndwQk1WVmtSSGRGUWk5M1VVVkJkMGxJWjBSQlZFSm5UbFpJVTFWRlJFUkJTMEpuWjNKQ1owVkdRbEZqUkVGNlFXUkNaMDVXU0ZFMFJVWm5VVlZqZFc5dkNtUnBZMngzWnpnd1dHZHRlako2U0dONFJXRlRVbU5OZDBoM1dVUldVakJxUWtKbmQwWnZRVlV6T1ZCd2VqRlphMFZhWWpWeFRtcHdTMFpYYVhocE5Ga0tXa1E0ZDJGM1dVUldVakJTUVZGSUwwSkhSWGRZTkZwa1lVaFNNR05JVFRaTWVUbHVZVmhTYjJSWFNYVlpNamwwVERKb01Wb3laSEJpYldSdFdWZE9iQXBNTW5Sc1kyMDFiR0pJVFhSWk1qbDBZbGhXZFdGWVVqVk1lVFZ1WVZoU2IyUlhTWFprTWpsNVlUSmFjMkl6WkhwTU1rb3hZVmQ0YTB4dWJHaGlWM2hCQ21OdFZtMWplVGx2V2xkR2EyTjVPWFJaVjJ4MVRVUnJSME5wYzBkQlVWRkNaemM0ZDBGUlJVVkxNbWd3WkVoQ2VrOXBPSFprUnpseVdsYzBkVmxYVGpBS1lWYzVkV041Tlc1aFdGSnZaRmRLTVdNeVZubFpNamwxWkVkV2RXUkROV3BpTWpCM1NIZFpTMHQzV1VKQ1FVZEVkbnBCUWtGblVWSmtNamw1WVRKYWN3cGlNMlJtV2tkc2VtTkhSakJaTW1kM1RtZFpTMHQzV1VKQ1FVZEVkbnBCUWtGM1VXOVBWMDB4V1ZSQ2JFMTZWbXhQVkdONlRUSldiRTlVVG10T1JGVTBDbHBFU210T1IxRXhUMFJGTUZwcVdUTmFSRWwzVDFkUmVVOVVRVlJDWjI5eVFtZEZSVUZaVHk5TlFVVkZRa0ZXUTJSWGJITmFSRUZ5UW1kdmNrSm5SVVVLUVZsUEwwMUJSVVpDUWpGdlpGZGtibUZYTlc1YWJVWnFXbE01Y2xwWVNuVmFWM2g2VEZkT2RtSlhNVEZpYld3d1pWUkJaRUpuYjNKQ1owVkZRVmxQTHdwTlFVVkhRa0U1ZVZwWFducE1NbWhzV1ZkU2Vrd3lNV2hoVnpSM1QzZFpTMHQzV1VKQ1FVZEVkbnBCUWtOQlVYUkVRM1J2WkVoU2QyTjZiM1pNTTFKMkNtRXlWblZNYlVacVpFZHNkbUp1VFhWYU1td3dZVWhXYVdSWVRteGpiVTUyWW01U2JHSnVVWFZaTWpsMFRVY3dSME5wYzBkQlVWRkNaemM0ZDBGUmEwVUtXSGQ0WkdGSVVqQmpTRTAyVEhrNWJtRllVbTlrVjBsMVdUSTVkRXd5YURGYU1tUndZbTFrYlZsWFRteE1NblJzWTIwMWJHSklUWFJaTWpsMFlsaFdkUXBoV0ZJMVRIazFibUZZVW05a1YwbDJaREk1ZVdFeVduTmlNMlI2VERKS01XRlhlR3RNYm14b1lsZDRRV050Vm0xamVUbHZXbGRHYTJONU9YUlpWMngxQ2sxRVowZERhWE5IUVZGUlFtYzNPSGRCVVc5RlMyZDNiMDlYVFRGWlZFSnNUWHBXYkU5VVkzcE5NbFpzVDFST2EwNUVWVFJhUkVwclRrZFJNVTlFUlRBS1dtcFpNMXBFU1hkUFYxRjVUMVJCWWtKbmIzSkNaMFZGUVZsUEwwMUJSVXhDUVRCTlF6Tk9iR0pIV1hSaFJ6bDZaRWRXYTAxRlFVZERhWE5IUVZGUlFncG5OemgzUVZGM1JVMW5kM2RoU0ZJd1kwaE5Oa3g1T1c1aFdGSnZaRmRKZFZreU9YUk1NbWd4V2pKa2NHSnRaRzFaVjA1c1RESjBiR050Tld4aVNFMTBDbGt5T1hSaVdGWjFZVmhTTlUxRVowZERhWE5IUVZGUlFtYzNPSGRCVVRCRlMyZDNiMDlYVFRGWlZFSnNUWHBXYkU5VVkzcE5NbFpzVDFST2EwNUVWVFFLV2tSS2EwNUhVVEZQUkVVd1dtcFpNMXBFU1hkUFYxRjVUMVJCWmtKbmIzSkNaMFZGUVZsUEwwMUJSVTlDUWtWTlJETktiRnB1VFhaaFIxWm9Xa2hOZGdwaVYwWndZbXBCWVVKbmIzSkNaMFZGUVZsUEwwMUJSVkJDUVhkTlEycEZkMDU2UlRCT2VsVXhUV3ByZDB4bldVdExkMWxDUWtGSFJIWjZRVUpGUVZGbkNrUkNOVzlrU0ZKM1kzcHZka3d5WkhCa1IyZ3hXV2sxYW1JeU1IWmhTRlp1V2pKc2RWb3lXbWhaTWxWM1IwRlpTMHQzV1VKQ1FVZEVkbnBCUWtWUlVVc0tSRUZuZVU1VVkzbE5SR013VFhwQ2RFSm5iM0pDWjBWRlFWbFBMMDFCUlZOQ1JqaE5XRmRvTUdSSVFucFBhVGgyV2pKc01HRklWbWxNYlU1MllsTTVid3BrVjJSdVlWYzFibHB0Um1wYVV6bHlXbGhLZFZwWGVIcE1WMDUyWWxjeE1XSnRiREJsVXpoMVdqSnNNR0ZJVm1sTU0yUjJZMjEwYldKSE9UTmplVGxwQ21SWGJITmFRelUxV1ZjeGMxRklTbXhhYmsxMllVZFdhRnBJVFhaaVYwWndZbXBCTkVKbmIzSkNaMFZGUVZsUEwwMUJSVlJDUTI5TlMwUnNhazVYUlhjS1dsUk5NVnBVYXpOTmVrNXNXbFJyZWxwRVVURlBSMUY1V2tSU2EwNVVaM2hPUjFreVRqSlJlVTFFYkd0TmFtdDNTVkZaUzB0M1dVSkNRVWRFZG5wQlFncEdRVkZVUkVKR00ySXpTbkphYlhoMlpERTVhMkZZVG5kWldGSnFZVVJDYTBKbmIzSkNaMFZGUVZsUEwwMUJSVlpDUmxsTlZrZG9NR1JJUW5wUGFUaDJDbG95YkRCaFNGWnBURzFPZG1KVE9XOWtWMlJ1WVZjMWJscHRSbXBhVXpseVdsaEtkVnBYZUhwTVYwNTJZbGN4TVdKdGJEQmxVemxvV1ROU2NHSXlOWG9LVEROS01XSnVUWFpOZWtWNFRVUm5lVTlVWjNkT1JGVjJXVmhTTUZwWE1YZGtTRTEyVFZSQlYwSm5iM0pDWjBWRlFWbFBMMDFCUlZkQ1FXZE5RbTVDTVFwWmJYaHdXWHBDUjBKbmIzSkNaMFZGUVZsUEwwMUJSVmxDUkdkTlRtNUtiR05IT0RaaFNGWnVXakpzZFZveVdtaFpNbFYyWVRKV2VXSnRWbk5qZVRGcUNtSXlNWFJrVnpWd1pFaHJObU50Vm0xUGJrcHNXbTVOZG1GSFZtaGFTRTEyWWxkR2NHSnFRMEpwWjFsTFMzZFpRa0pCU0ZkbFVVbEZRV2RTT0VKSWIwRUtaVUZDTWtGT01EbE5SM0pIZUhoRmVWbDRhMlZJU214dVRuZExhVk5zTmpRemFubDBMelJsUzJOdlFYWkxaVFpQUVVGQlFtNDVaR3BMWWtWQlFVRlJSQXBCUldOM1VsRkphRUZQYzJsT1RsTm1kak5rV0RjM2NXTk9PV2wxVGxwcU1tOVVjR3Q1VWt3emQwRjVUa2hVUjB0NFdtVXdRV2xDVURGS1dDdHRPWEowQ21wUVlsbEZXRk5HT0VOM0syaHJOVUp5V2pWbWVIcE9hVnBETTFSSk9VWm1aMnBCUzBKblozRm9hMnBQVUZGUlJFRjNUbTlCUkVKc1FXcEZRUzl6T1RVS2QzZEdVa1JSVlU5WU1EWlFTMVpGZERRMVN6WlNibWhUSzBGVmJUY3dMMHhKV0hGaWNVVXJOR3RHYWxjMWNuVkdlVllyY0c5MFRYaE1XVXRaUVdwQ1lRcHZWRE16WjJGWUx6SmpibEF2UlhNMWJVSnpOMVZDWVdKVmJTOURaa1paVEN0RFFtWXdUM3BHUzAxWWMxbHVNMGRpSzFaU1prRm5hSFZZZUZrcmNVazlDaTB0TFMwdFJVNUVJRU5GVWxSSlJrbERRVlJGTFMwdExTMEsifX19fQ=="}], "timestampVerificationData":{"rfc3161Timestamps":[{"signedTimestamp":"MIICyDADAgEAMIICvwYJKoZIhvcNAQcCoIICsDCCAqwCAQMxDTALBglghkgBZQMEAgEwgbcGCyqGSIb3DQEJEAEEoIGnBIGkMIGhAgEBBgkrBgEEAYO/MAIwMTANBglghkgBZQMEAgEFAAQgh4l2yHCu2Fcn5cUOXzkBeFDycvtXlAwWfstiCWJHOBICFCoS8RsGtrqrA0MYbJJ1ijGKLC3/GA8yMDI2MDgwNjE0MDM0OFowAwIBAaAypDAwLjEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MRUwEwYDVQQDEwxzaWdzdG9yZS10c2GgADGCAdowggHWAgEBMFEwOTEVMBMGA1UEChMMc2lnc3RvcmUuZGV2MSAwHgYDVQQDExdzaWdzdG9yZS10c2Etc2VsZnNpZ25lZAIUOhNULwyQYe68wUMvy4qOiyojiwwwCwYJYIZIAWUDBAIBoIH8MBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAcBgkqhkiG9w0BCQUxDxcNMjYwODA2MTQwMzQ4WjAvBgkqhkiG9w0BCQQxIgQg01rCf9QOSiyZ4ePyHpLBqsF0VteUJpog/dh5HI/aZF0wgY4GCyqGSIb3DQEJEAIvMX8wfTB7MHkEIIX5J7wHq2LKw7RDVsEO/IGyxog/2nq55thw2dE6zQW3MFUwPaQ7MDkxFTATBgNVBAoTDHNpZ3N0b3JlLmRldjEgMB4GA1UEAxMXc2lnc3RvcmUtdHNhLXNlbGZzaWduZWQCFDoTVC8MkGHuvMFDL8uKjosqI4sMMAoGCCqGSM49BAMCBGYwZAIwPjTPW5DMjgqk0A64hqM6XisNWoNvg6jPTrOWBB56tPdRKkapJd0uAl8aFjYnhRM2AjBGOab1G98aPMpxMrVAJ60YS1Pf0MzpeZhaYTbUf/nREyNTRYeuZ9V30AkZlQ7CYnQ="}]}}, "messageSignature":{"messageDigest":{"algorithm":"SHA2_256", "digest":"FOHiyGfALV5PKjwsZlJ5cWXb85qG+pDJttCfy9Xt7uw="}, "signature":"MEYCIQC+IRtf+n7O44UK2BN6Y2nN17hgiCpPzHFomjID/Jt+SgIhAOzeq0EFMVDbDb+/TUbOzTAgQXXBA//QDXKkUl5AxCC9"}}

build/torch-stable-abi29-cu132-x86_64-linux/nvfp4_gemm/__init__.py ADDED Viewed

	@@ -0,0 +1,26 @@

+import ctypes
+import importlib.util
+import sys
+from pathlib import Path
+from types import ModuleType
+def _import_from_path(file_path: Path) -> ModuleType:
+    # We cannot use the module name as-is, after adding it to `sys.modules`,
+    # it would also be used for other imports. So, we make a module name that
+    # depends on the path for it to be unique using the hex-encoded hash of
+    # the path.
+    path_hash = "{:x}".format(ctypes.c_size_t(hash(file_path.absolute())).value)
+    module_name = path_hash
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if spec is None:
+        raise ImportError(f"Cannot load spec for {module_name} from {file_path}")
+    module = importlib.util.module_from_spec(spec)
+    if module is None:
+        raise ImportError(f"Cannot load module {module_name} from spec")
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)  # type: ignore
+    return module
+globals().update(vars(_import_from_path(Path(__file__).parent.parent / "__init__.py")))