Scaffold-GS: Structured 3D Gaussians, Sparse Voxel Anchors, and Model Compression

Step 1: Sparse Voxel Anchor Grid Container

from __future__ import annotations

from typing import Dict, Tuple, List
import torch
import torch.nn as nn


class SparseAnchorGrid(nn.Module):
    def __init__(
        self,
        voxel_size: float = 0.3,
        feature_dim: int = 32,
        max_gaussians_per_anchor: int = 4,
        device: str = "cuda",
    ):
        super().__init__()
        self.voxel_size = voxel_size
        self.feature_dim = feature_dim
        self.max_gaussians = max_gaussians_per_anchor
        self.device = device

        self.anchor_positions = torch.zeros((0, 3), device=device)
        self.anchor_features = nn.Parameter(torch.zeros((0, feature_dim), device=device))
        self.voxel_map: Dict[Tuple[int, int, int], int] = {}

    def populate_from_points(self, pcd_points: torch.Tensor) -> SparseAnchorGrid:
        """Initializes sparse anchors from SfM point coordinates."""
        voxel_coords = torch.floor(pcd_points / self.voxel_size).long()
        unique_voxels, inverse_indices = torch.unique(voxel_coords, dim=0, return_inverse=True)

        num_anchors = unique_voxels.shape[0]
        anchor_pos = (unique_voxels.float() + 0.5) * self.voxel_size

        self.anchor_positions = anchor_pos.to(self.device)
        self.anchor_features = nn.Parameter(
            torch.randn(num_anchors, self.feature_dim, device=self.device) * 0.01
        )

        for idx, vox in enumerate(unique_voxels.tolist()):
            self.voxel_map[tuple(vox)] = idx

        return self

    def query_nearest_anchors(self, query_pts: torch.Tensor, k: int = 1) -> Tuple[torch.Tensor, torch.Tensor]:
        """Queries nearest anchors for query coordinates."""
        dists = torch.cdist(query_pts, self.anchor_positions)
        min_dists, min_idx = torch.topk(dists, k=k, dim=1, largest=False)

        selected_positions = self.anchor_positions[min_idx[:, 0]]
        selected_features = self.anchor_features[min_idx[:, 0]]
        return selected_positions, selected_features

Step 2: Shared Neural Predictor (MLP Decoder)

from __future__ import annotations

import torch
import torch.nn as nn
import torch.nn.functional as F


class SharedGaussianMLP(nn.Module):
    """Shared neural decoder mapping anchor features to local Gaussian primitives."""

    def __init__(
        self,
        feature_dim: int = 32,
        num_gaussians: int = 4,
        hidden_dim: int = 64,
        sh_dim: int = 48,  # Degree 3 SH (16 x 3)
    ):
        super().__init__()
        self.num_gaussians = num_gaussians
        self.out_dim_per_gaussian = 3 + 3 + 4 + 1 + sh_dim  # (d_pos, scale, quat, opacity, SH)

        input_dim = feature_dim + 3 + 3  # (feature + delta_x + view_direction)

        self.backbone = nn.Sequential(
            nn.Linear(input_dim, hidden_dim),
            nn.GELU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.GELU(),
            nn.Linear(hidden_dim, num_gaussians * self.out_dim_per_gaussian),
        )

    def forward(
        self,
        anchor_feat: torch.Tensor,
        delta_x: torch.Tensor,
        view_dirs: torch.Tensor,
    ) -> Dict[str, torch.Tensor]:
        """anchor_feat: (N, D), delta_x: (N, 3), view_dirs: (N, 3)."""
        x = torch.cat([anchor_feat, delta_x, view_dirs], dim=-1)
        raw_out = self.backbone(x).view(-1, self.num_gaussians, self.out_dim_per_gaussian)

        pos_offset = raw_out[..., 0:3]
        scale = torch.exp(torch.clamp(raw_out[..., 3:6], min=-7.0, max=2.0))
        quat = F.normalize(raw_out[..., 6:10], dim=-1)
        opacity = torch.sigmoid(raw_out[..., 10:11])
        sh_coeffs = raw_out[..., 11:]

        return {
            "pos_offsets": pos_offset,
            "scales": scale,
            "rotations": quat,
            "opacities": opacity,
            "sh_coeffs": sh_coeffs,
        }

Step 3: Level-of-Detail (LOD) Distance Hierarchical Filter

from __future__ import annotations

import torch
from anchor_grid import SparseAnchorGrid
from mlp_predictor import SharedGaussianMLP
from gsplat import rasterization


class LODScaffoldRenderer:
    def __init__(self, anchor_grid: SparseAnchorGrid, mlp: SharedGaussianMLP):
        self.anchor_grid = anchor_grid
        self.mlp = mlp

    @torch.no_grad()
    def render_lod(
        self,
        camera_pos: torch.Tensor,
        viewmat: torch.Tensor,
        K: torch.Tensor,
        width: int = 800,
        height: int = 800,
        lod_cutoff_distance: float = 8.0,
    ) -> torch.Tensor:
        anchors = self.anchor_grid.anchor_positions
        features = self.anchor_grid.anchor_features

        # Filter out anchors beyond LOD cutoff distance
        dists = torch.norm(anchors - camera_pos, dim=-1)
        active_mask = dists < lod_cutoff_distance

        active_anchors = anchors[active_mask]
        active_features = features[active_mask]

        if active_anchors.shape[0] == 0:
            return torch.zeros((3, height, width), device=camera_pos.device)

        view_dirs = F.normalize(active_anchors - camera_pos, dim=-1)
        delta_zeros = torch.zeros_like(active_anchors)

        decoded = self.mlp(active_features, delta_zeros, view_dirs)

        means = (active_anchors.unsqueeze(1) + decoded["pos_offsets"]).view(-1, 3)
        scales = decoded["scales"].view(-1, 3)
        quats = decoded["rotations"].view(-1, 4)
        opacities = decoded["opacities"].view(-1, 1)
        sh_colors = decoded["sh_coeffs"].view(-1, 16, 3)

        rendered, _, _ = rasterization(
            means=means[None],
            quats=quats[None],
            scales=scales[None],
            opacities=opacities[None],
            colors=sh_colors[None],
            viewmats=viewmat[None],
            Ks=K[None],
            width=width,
            height=height,
            sh_degree=3,
            render_mode="RGB",
            near_plane=0.01,
            far_plane=100.0,
            tile_size=16,
            packed=True,
        )
        return rendered[0]

Empirical Benchmark Evaluation

We evaluated Scaffold-GS against state-of-the-art compact radiance field representations on the Mip-NeRF 360 benchmark:

Troubleshooting Common Scaffold-GS Artifacts

1. Discontinuous Seams Across Voxel Boundaries

• Symptom: Visible grid boundary artifacts when moving camera across voxel chart boundaries.
• Remedy: Query $k=3$ nearest anchors and blend local Gaussian predictions via inverse-distance softmax weights.

2. Feature Vector Quantization Degradation

• Symptom: Noticeable color banding or surface blurring post Product Quantization.
• Remedy: Increase the number of sub-vector quantization splits from $M=4$ to $M=8$ and retrain codebook centroids on the active feature manifold.

3. Voxel Grid Sparsity Collapse in High-Detail Regions

• Symptom: Fine wire or plant foliage structures fail to resolve.
• Remedy: Reduce the base voxel size $v_{\text{size}}$ from $0.3\text{ m}$ to $0.15\text{ m}$ in dense geometric regions.