Text-to-3D Gaussian Splatting: Score Distillation Sampling, Diffusion Priors, and PyTorch Training

Step 1: SDXL Diffusion Guidance Engine

from __future__ import annotations

import torch
import torch.nn as nn
from diffusers import StableDiffusionXLPipeline


class SDXLScoreGuidance(nn.Module):
    """Wraps pre-trained SDXL for Score Distillation Sampling."""

    def __init__(
        self,
        model_id: str = "stabilityai/stable-diffusion-xl-base-1.0",
        device: str = "cuda",
        dtype: torch.dtype = torch.float16,
    ):
        super().__init__()
        self.device = device
        self.dtype = dtype

        pipe = StableDiffusionXLPipeline.from_pretrained(
            model_id, torch_dtype=dtype, use_safetensors=True
        )
        self.unet = pipe.unet.to(device).eval()
        self.vae = pipe.vae.to(device).eval()
        self.scheduler = pipe.scheduler
        self.tokenizer = pipe.tokenizer
        self.tokenizer_2 = pipe.tokenizer_2
        self.text_encoder = pipe.text_encoder.to(device).eval()
        self.text_encoder_2 = pipe.text_encoder_2.to(device).eval()

        # Freeze all backbone weights
        for p in self.parameters():
            p.requires_grad = False

    @torch.no_grad()
    def encode_prompt(self, prompt: str) -> Tuple[torch.Tensor, torch.Tensor]:
        """Extracts text embeddings and pooled projections from SDXL text encoders."""
        inputs_1 = self.tokenizer(prompt, padding="max_length", max_length=77, return_tensors="pt").input_ids.to(self.device)
        inputs_2 = self.tokenizer_2(prompt, padding="max_length", max_length=77, return_tensors="pt").input_ids.to(self.device)

        hidden_1 = self.text_encoder(inputs_1)[0]
        hidden_2 = self.text_encoder_2(inputs_2, output_hidden_states=True)
        pooled = hidden_2.text_embeds
        hidden_2 = hidden_2.hidden_states[-2]

        prompt_embeds = torch.cat([hidden_1, hidden_2], dim=-1)
        return prompt_embeds, pooled

    def compute_sds_gradient(
        self,
        rendered_images: torch.Tensor,
        prompt_embeds: torch.Tensor,
        pooled_embeds: torch.Tensor,
        guidance_scale: float = 50.0,
        t_min_ratio: float = 0.02,
        t_max_ratio: float = 0.98,
    ) -> torch.Tensor:
        """Computes dL_SDS / d(image) via VAE encode and U-Net noise residual."""
        # 1. Encode rendered RGB [-1, 1] into VAE latent space
        latents = self.vae.encode(rendered_images.to(self.dtype)).latent_dist.sample()
        latents = latents * self.vae.config.scaling_factor

        batch_size = latents.shape[0]
        num_train_timesteps = self.scheduler.config.num_train_timesteps
        t = torch.randint(
            int(t_min_ratio * num_train_timesteps),
            int(t_max_ratio * num_train_timesteps),
            (batch_size,),
            device=self.device,
        ).long()

        noise = torch.randn_like(latents)
        noisy_latents = self.scheduler.add_noise(latents, noise, t)

        # 2. Duplicate inputs for Classifier-Free Guidance (CFG)
        latent_model_input = torch.cat([noisy_latents, noisy_latents], dim=0)
        t_input = torch.cat([t, t], dim=0)
        uncond_embeds = torch.zeros_like(prompt_embeds)
        uncond_pooled = torch.zeros_like(pooled_embeds)

        context = torch.cat([uncond_embeds, prompt_embeds], dim=0)
        pooled_context = torch.cat([uncond_pooled, pooled_embeds], dim=0)

        # Added time IDs for SDXL resolution conditioning
        add_time_ids = torch.tensor(
            [[512.0, 512.0, 0.0, 0.0, 512.0, 512.0]], device=self.device, dtype=self.dtype
        ).repeat(batch_size * 2, 1)

        added_cond_kwargs = {"text_embeds": pooled_context, "time_ids": add_time_ids}

        with torch.no_grad():
            noise_pred = self.unet(
                latent_model_input,
                t_input,
                encoder_hidden_states=context,
                added_cond_kwargs=added_cond_kwargs,
            ).sample

        noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
        noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)

        # 3. Compute score residual
        w = 1.0 - (t.float() / num_train_timesteps).view(-1, 1, 1, 1)
        grad_latent = w * (noise_pred - noise)

        # Decode latent gradient back to RGB space
        grad_image = self.vae.decode(grad_latent / self.vae.config.scaling_factor).sample
        return grad_image.float()

Step 2: Camera Sampling & Differentiable Forward Pipeline

from __future__ import annotations

import math
import torch


def sample_random_cameras(
    batch_size: int,
    image_size: int = 512,
    radius_range: Tuple[float, float] = (1.8, 2.5),
    elevation_range: Tuple[float, float] = (-20.0, 45.0),
    azimuth_range: Tuple[float, float] = (-180.0, 180.0),
    device: str = "cuda",
) -> Tuple[torch.Tensor, torch.Tensor]:
    """Uniformly samples camera poses on a sphere with look-at extrinsics."""
    extrinsics, intrinsics = [], []

    for _ in range(batch_size):
        azimuth = math.radians(torch.empty(1).uniform_(*azimuth_range).item())
        elevation = math.radians(torch.empty(1).uniform_(*elevation_range).item())
        radius = torch.empty(1).uniform_(*radius_range).item()

        # Spherical to Cartesian
        cam_x = radius * math.cos(elevation) * math.sin(azimuth)
        cam_y = radius * math.sin(elevation)
        cam_z = radius * math.cos(elevation) * math.cos(azimuth)
        cam_pos = torch.tensor([cam_x, cam_y, cam_z], dtype=torch.float32)

        # Look-at matrix (toward origin)
        forward = -cam_pos / torch.norm(cam_pos)
        up = torch.tensor([0.0, 1.0, 0.0])
        right = torch.cross(forward, up)
        right = right / torch.norm(right)
        up = torch.cross(right, forward)

        w2c = torch.eye(4)
        w2c[:3, 0] = right
        w2c[:3, 1] = up
        w2c[:3, 2] = -forward
        w2c[:3, 3] = cam_pos
        extrinsics.append(w2c)

        # Intrinsics with 50-degree FoV
        fov = math.radians(50.0)
        focal = 0.5 * image_size / math.tan(fov / 2.0)
        K = torch.tensor([
            [focal, 0.0, image_size / 2.0],
            [0.0, focal, image_size / 2.0],
            [0.0, 0.0, 1.0],
        ])
        intrinsics.append(K)

    return torch.stack(extrinsics).to(device), torch.stack(intrinsics).to(device)

Step 3: Two-Stage Optimization Engine

from __future__ import annotations

import torch
import torch.nn as nn
from tqdm import tqdm
from gsplat import rasterization
from sdxl_guidance import SDXLScoreGuidance
from camera_sampler import sample_random_cameras


class TextTo3DGSTrainer:
    def __init__(self, prompt: str, num_gaussians: int = 10_000, device: str = "cuda"):
        self.device = device
        self.guidance = SDXLScoreGuidance(device=device)
        self.prompt_embeds, self.pooled_embeds = self.guidance.encode_prompt(prompt)

        # Initialize spherical Gaussian volume
        positions = torch.randn(num_gaussians, 3, device=device) * 0.2
        scales = torch.full((num_gaussians, 3), -4.0, device=device)  # exp(-4) ~ 0.018
        quats = torch.zeros(num_gaussians, 4, device=device)
        quats[:, 0] = 1.0
        opacities = torch.logit(torch.full((num_gaussians, 1), 0.1, device=device))
        sh_dc = torch.randn(num_gaussians, 1, 3, device=device) * 0.1

        self.xyz = nn.Parameter(positions.requires_grad_(True))
        self.scaling = nn.Parameter(scales.requires_grad_(True))
        self.rotation = nn.Parameter(quats.requires_grad_(True))
        self.opacity = nn.Parameter(opacities.requires_grad_(True))
        self.features_dc = nn.Parameter(sh_dc.requires_grad_(True))

        self.optimizer = torch.optim.Adam([
            {"params": [self.xyz], "lr": 0.001},
            {"params": [self.scaling], "lr": 0.005},
            {"params": [self.rotation], "lr": 0.002},
            {"params": [self.opacity], "lr": 0.02},
            {"params": [self.features_dc], "lr": 0.01},
        ])

    def train_step(self, step: int, batch_views: int = 4, image_size: int = 512) -> float:
        self.optimizer.zero_grad()
        viewmats, Ks = sample_random_cameras(batch_views, image_size=image_size, device=self.device)

        # Forward pass through gsplat
        rendered_frames = []
        for i in range(batch_views):
            rendered, _, _ = rasterization(
                means=self.xyz[None],
                quats=torch.nn.functional.normalize(self.rotation[None], dim=-1),
                scales=torch.exp(self.scaling[None]),
                opacities=torch.sigmoid(self.opacity[None]),
                colors=self.features_dc[None],
                viewmats=viewmats[i : i + 1],
                Ks=Ks[i : i + 1],
                width=image_size,
                height=image_size,
                sh_degree=0,
                render_mode="RGB",
                near_plane=0.01,
                far_plane=100.0,
                packed=True,
            )
            rendered_frames.append(rendered[0].permute(2, 0, 1))

        batch_rendered = torch.stack(rendered_frames)  # (B, 3, H, W) in [0, 1]
        batch_input = batch_rendered * 2.0 - 1.0       # Map to [-1, 1] for SDXL

        # Compute SDS target gradients
        grad_image = self.guidance.compute_sds_gradient(
            batch_input,
            self.prompt_embeds.repeat(batch_views, 1, 1),
            self.pooled_embeds.repeat(batch_views, 1),
            guidance_scale=50.0,
        )

        # Backpropagate through differentiable renderer
        loss = (grad_image * batch_input).sum()
        loss.backward()

        torch.nn.utils.clip_grad_norm_([self.xyz, self.scaling, self.rotation, self.features_dc], 1.0)
        self.optimizer.step()
        return loss.item()

Empirical Benchmark Evaluation

We evaluated Text-to-3DGS generation quality and latency across the T3Bench text-to-3D benchmark:

Troubleshooting Common Distillation Artifacts

1. Multi-Face “Janus” Problem

• Symptom: Objects exhibit identical duplicate faces or features on the back and sides.
• Remedy: Incorporate viewpoint-dependent text prompting (append front view, side view, back view conditioned on sampled camera azimuth $\varphi$).

2. High-Frequency Color Oversaturation

• Symptom: Acidic, blown-out colors and burned contrast.
• Remedy: Reduce guidance scale $s_{\text{cfg}}$ from $100\to 50$ during the first 1,000 steps and enable latent gradient norm clipping.

3. Hollow Geometry and Floating Splats

• Symptom: Gaussians fail to coalesce into solid surfaces.
• Remedy: Add a monocular depth regularization loss ${\mathcal{L}}_{\text{depth}}=\Vert \nabla D_{\text{render}}-\nabla D_{\text{MiDaS}}{\Vert }_1$ using a frozen depth estimator.