4D Gaussian Splatting for Dynamic Scenes: Canonical Fields, HexPlane Deformation, and Real-Time Rendering

Step 1: Canonical Gaussian Container

from __future__ import annotations

import math
import torch
import torch.nn as nn


class CanonicalGaussians(nn.Module):
    """Time-invariant canonical 3D Gaussian container."""

    def __init__(self, init_positions: torch.Tensor, max_sh_degree: int = 2):
        super().__init__()
        num_points = init_positions.shape[0]
        self.max_sh_degree = max_sh_degree

        self.positions = nn.Parameter(init_positions.clone())
        self.scales = nn.Parameter(torch.full((num_points, 3), math.log(0.02)))
        self.rotations = nn.Parameter(torch.zeros(num_points, 4))
        self.opacities = nn.Parameter(torch.full((num_points, 1), float(np.log(0.1 / 0.9))))

        with torch.no_grad():
            self.rotations[:, 0] = 1.0  # Unit quaternion [w, x, y, z]

        sh_dims = (max_sh_degree + 1) ** 2
        self.sh_coeffs = nn.Parameter(torch.zeros(num_points, sh_dims, 3))

    def get_xyz(self) -> torch.Tensor:
        return self.positions

    def get_scaling(self) -> torch.Tensor:
        return torch.exp(self.scales)

    def get_rotation(self) -> torch.Tensor:
        return torch.nn.functional.normalize(self.rotations, dim=-1)

    def get_opacity(self) -> torch.Tensor:
        return torch.sigmoid(self.opacities)

    def get_features(self) -> torch.Tensor:
        return self.sh_coeffs

Step 2: HexPlane Space-Time Deformation Field

from __future__ import annotations

from typing import Tuple, Dict
import torch
import torch.nn as nn
import torch.nn.functional as F


class HexPlaneDeformation(nn.Module):
    """HexPlane feature decomposition with MLP decoding head."""

    def __init__(
        self,
        bounds: float = 1.2,
        plane_res: int = 64,
        plane_dim: int = 32,
        hidden_dim: int = 128,
    ):
        super().__init__()
        self.bounds = bounds
        self.plane_keys = ["xy", "xz", "yz", "xt", "yt", "zt"]

        # Initialize six orthogonal 2D feature grids
        self.planes = nn.ParameterDict({
            k: nn.Parameter(torch.randn(plane_res, plane_res, plane_dim) * 0.01)
            for k in self.plane_keys
        })

        self.decoder = nn.Sequential(
            nn.Linear(6 * plane_dim, hidden_dim),
            nn.SiLU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.SiLU(),
        )

        self.head_xyz = nn.Linear(hidden_dim, 3)
        self.head_rot = nn.Linear(hidden_dim, 4)
        self.head_scale = nn.Linear(hidden_dim, 3)

        # Zero-initialize heads for identity deformation at cold start
        for head in [self.head_xyz, self.head_rot, self.head_scale]:
            nn.init.zeros_(head.weight)
            nn.init.zeros_(head.bias)

    def _sample_grid(self, plane: torch.Tensor, uv: torch.Tensor) -> torch.Tensor:
        num_pts = uv.shape[0]
        grid = plane.permute(2, 0, 1)[None]  # (1, C, H, W)
        sampled = F.grid_sample(
            grid, uv.view(1, num_pts, 1, 2), mode="bilinear", align_corners=True
        )
        return sampled.squeeze(-1).squeeze(0).permute(1, 0)

    def forward(self, mu_c: torch.Tensor, t: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        num_pts = mu_c.shape[0]
        if t.ndim == 1:
            t = t.unsqueeze(0).expand(num_pts, 1)

        norm_xyz = torch.clamp(mu_c / self.bounds, -1.0, 1.0)
        norm_t = torch.clamp(t * 2.0 - 1.0, -1.0, 1.0)

        coords: Dict[str, torch.Tensor] = {
            "xy": torch.cat([norm_xyz[:, 0:1], norm_xyz[:, 1:2]], dim=-1),
            "xz": torch.cat([norm_xyz[:, 0:1], norm_xyz[:, 2:3]], dim=-1),
            "yz": torch.cat([norm_xyz[:, 1:2], norm_xyz[:, 2:3]], dim=-1),
            "xt": torch.cat([norm_xyz[:, 0:1], norm_t], dim=-1),
            "yt": torch.cat([norm_xyz[:, 1:2], norm_t], dim=-1),
            "zt": torch.cat([norm_xyz[:, 2:3], norm_t], dim=-1),
        }

        features = torch.cat([self._sample_grid(self.planes[k], coords[k]) for k in self.plane_keys], dim=-1)
        latent = self.decoder(features)

        d_xyz = self.head_xyz(latent)
        d_rot = self.head_rot(latent)
        d_scale = self.head_scale(latent)
        return d_xyz, d_rot, d_scale

Step 3: Integrated 4DGS Model & Differentiable Forward Pass

from __future__ import annotations

from typing import Tuple
import torch
import torch.nn as nn
import torch.nn.functional as F

from .canonical_gaussians import CanonicalGaussians
from .deformation_hexplane import HexPlaneDeformation


class FourDGS(nn.Module):
    def __init__(self, init_positions: torch.Tensor, max_sh_degree: int = 2):
        super().__init__()
        self.canonical = CanonicalGaussians(init_positions, max_sh_degree=max_sh_degree)
        self.deformation = HexPlaneDeformation()

    def forward(self, t: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
        mu_c = self.canonical.get_xyz()
        s_c = self.canonical.get_scaling()
        q_c = self.canonical.get_rotation()
        alpha = self.canonical.get_opacity()
        sh = self.canonical.get_features()

        d_xyz, d_rot, d_scale = self.deformation(mu_c, t)

        mu_t = mu_c + d_xyz
        q_t = F.normalize(q_c + 0.1 * d_rot, dim=-1)
        s_t = s_c * torch.exp(0.05 * d_scale)

        return mu_t, s_t, q_t, alpha, sh

Step 4: Training Loop with Isometry Regularization

from __future__ import annotations

import torch
import gsplat
from model.four_d_gs import FourDGS


def compute_isometry_loss(mu_c: torch.Tensor, d_xyz: torch.Tensor, k: int = 8) -> torch.Tensor:
    """Evaluates local deformation variance across k-nearest spatial neighbors."""
    with torch.no_grad():
        dists = torch.cdist(mu_c, mu_c)
        knn_indices = dists.topk(k + 1, largest=False).indices[:, 1:]

    d_xyz_neighbors = d_xyz[knn_indices]  # (N, k, 3)
    variance = d_xyz_neighbors.var(dim=1).mean()
    return variance


def train_step(
    model: FourDGS,
    optimizer: torch.optim.Optimizer,
    batch: dict,
    width: int,
    height: int,
    lambda_iso: float = 0.2,
) -> float:
    optimizer.zero_grad()
    t = batch["timestamp"]
    w2c = batch["w2c"]
    k_mat = batch["intrinsics"]
    gt_rgb = batch["image"]

    mu_t, s_t, q_t, alpha, sh = model(t)

    rendered, _, _ = gsplat.rasterization(
        means=mu_t,
        quats=q_t,
        scales=s_t,
        opacities=alpha.squeeze(-1),
        colors=sh,
        viewmats=w2c[None],
        Ks=k_mat[None],
        width=width,
        height=height,
        sh_degree=model.canonical.max_sh_degree,
        packed=False,
    )

    pred_rgb = rendered[0].permute(2, 0, 1)
    loss_photo = F.l1_loss(pred_rgb, gt_rgb)

    d_xyz, _, _ = model.deformation(model.canonical.get_xyz(), t)
    loss_iso = compute_isometry_loss(model.canonical.get_xyz(), d_xyz)

    total_loss = loss_photo + lambda_iso * loss_iso
    total_loss.backward()
    optimizer.step()

    return total_loss.item()

Empirical Benchmark Results

We benchmarked 4DGS against Dynamic NeRF and Per-Frame 3DGS across the DyNeRF and Neural 3D Video public datasets:

Figure 4: Qualitative novel-view synthesis. 4DGS eliminates the temporal ghosting and high-frequency flicker characteristic of unconstrained per-frame reconstructions.

Real-Time Production & Robotics Integration

1. Inference Buffer Pre-Baking: For deterministic replay, pre-evaluating $\Phi ({\mu }^c,t)$ over $T$ steps converts the sequence into fixed CUDA memory buffers, achieving identical rendering throughput to static 3DGS (> 300 FPS).
2. Predictive Digital Twins: By querying $t+\Delta t$, robotic trajectory planners predict obstacle movement up to 200 ms ahead for proactive collision avoidance.
3. Sim-to-Real Domain Transfer: Dynamic 4DGS models captured from physical human demonstrations can be imported directly into simulators (Isaac Sim, MuJoCo) as dynamic interactive collision environments.

Troubleshooting Common Deployment Pitfalls

1. “Elastic Jelly” Distortion

• Root Cause: Insufficient geometric constraints cause Gaussians to move independently to overfit 2D pixels.
• Solution: Increase ${\lambda }_{\text{iso}}$ to $0.4\sim 0.8$ and enforce learning rate decay on the deformation MLP.

2. Temporal Ghosting on Rapid Motions

• Root Cause: Temporal frequency of HexPlane is too low to resolve high-speed acceleration.
• Solution: Increase temporal plane resolution $R_t$ from 64 to 128 and initialize canonical positions from the sequence’s median frame.

3. VRAM Accumulation During Long Sequences

• Root Cause: Retaining computation graphs across batch iterations.
• Solution: Decouple temporal evaluation into sliding sub-windows ($T\le 150$ frames) and train across hierarchically linked keyframe anchors.