Microsoft TRELLIS: Structured 3D Latents (SLAT), Rectified Flow Transformers, and Unified Multi-Format 3D Generation

Step 1: Loading Pre-trained Multi-Modal Pipelines

from __future__ import annotations

from typing import Optional, Dict, Any
import torch
from PIL import Image
from trellis.pipelines import TrellisImageTo3DPipeline, TrellisTextTo3DPipeline


class TrellisInferenceEngine:
    def __init__(
        self,
        image_model_id: str = "microsoft/TRELLIS-image-large",
        text_model_id: str = "microsoft/TRELLIS-text-xlarge",
        device: str = "cuda",
        dtype: torch.dtype = torch.bfloat16,
    ):
        self.device = device
        self.dtype = dtype

        # Initialize Image-to-3D pipeline
        self.image_pipeline = TrellisImageTo3DPipeline.from_pretrained(
            image_model_id, torch_dtype=dtype
        ).to(device)
        self.image_pipeline.enable_model_cpu_offload()

        # Initialize Text-to-3D pipeline lazily when requested
        self._text_pipeline: Optional[TrellisTextTo3DPipeline] = None
        self._text_model_id = text_model_id

    @property
    def text_pipeline(self) -> TrellisTextTo3DPipeline:
        if self._text_pipeline is None:
            self._text_pipeline = TrellisTextTo3DPipeline.from_pretrained(
                self._text_model_id, torch_dtype=self.dtype
            ).to(self.device)
            self._text_pipeline.enable_model_cpu_offload()
        return self._text_pipeline

    def generate_from_image(
        self,
        image_path: str,
        num_inference_steps: int = 30,
        cfg_strength: float = 7.5,
    ) -> Dict[str, Any]:
        """Runs feed-forward generation from a single image."""
        raw_image = Image.open(image_path).convert("RGBA")

        # Execute flow matching across sparse structure and SLAT
        outputs = self.image_pipeline(
            raw_image,
            num_inference_steps=num_inference_steps,
            guidance_scale=cfg_strength,
            sparse_structure_sampler_params={"steps": 12, "cfg_strength": 7.5},
            slat_sampler_params={"steps": 12, "cfg_strength": 3.0},
        )
        return {
            "gs": outputs.decoded_gs,
            "mesh": outputs.decoded_mesh,
            "nerf": outputs.decoded_rf,
            "slat": outputs.slat,
        }

    def generate_from_text(
        self,
        prompt: str,
        num_inference_steps: int = 40,
        cfg_strength: float = 8.0,
    ) -> Dict[str, Any]:
        """Runs text-conditioned 3D flow matching."""
        outputs = self.text_pipeline(
            prompt,
            num_inference_steps=num_inference_steps,
            guidance_scale=cfg_strength,
            sparse_structure_sampler_params={"steps": 16, "cfg_strength": 8.0},
            slat_sampler_params={"steps": 16, "cfg_strength": 4.0},
        )
        return {
            "gs": outputs.decoded_gs,
            "mesh": outputs.decoded_mesh,
            "nerf": outputs.decoded_rf,
            "slat": outputs.slat,
        }

Step 2: Production Asset Export & Mesh Topology Optimization

from __future__ import annotations

import trimesh
import open3d as o3d
import numpy as np


def postprocess_and_export_mesh(
    raw_mesh: trimesh.Trimesh,
    output_glb_path: str,
    target_triangles: int = 40_000,
) -> trimesh.Trimesh:
    """Cleans non-manifold geometry, simplifies mesh, and exports to GLTF."""
    # 1. Extract largest connected component
    components = raw_mesh.split(only_watertight=False)
    components.sort(key=lambda c: -len(c.vertices))
    cleaned_mesh = components[0]

    # 2. Quadric decimation via Open3D
    o3d_mesh = o3d.geometry.TriangleMesh(
        vertices=o3d.utility.Vector3dVector(cleaned_mesh.vertices),
        triangles=o3d.utility.Vector3iVector(cleaned_mesh.faces),
    )
    o3d_mesh = o3d_mesh.remove_duplicated_vertices()
    o3d_mesh = o3d_mesh.remove_degenerate_triangles()
    o3d_mesh = o3d_mesh.compute_vertex_normals()

    decimated = o3d_mesh.simplify_quadric_decimation(target_number_of_triangles=target_triangles)
    decimated = decimated.filter_smooth_laplacian(number_of_iterations=2)

    final_mesh = trimesh.Trimesh(
        vertices=np.asarray(decimated.vertices),
        faces=np.asarray(decimated.triangles),
        process=True,
    )
    final_mesh.export(output_glb_path)
    return final_mesh


def export_gaussian_splat(decoded_gs: Any, output_ply_path: str) -> None:
    """Saves generated 3D Gaussian Splat primitives to standard PLY format."""
    decoded_gs.save_ply(output_ply_path)

Step 3: Localized Latent Inpainting (SLAT Spatial Editing)

from __future__ import annotations

import torch
from typing import Tuple


def inpaint_slat_bounding_box(
    engine: TrellisInferenceEngine,
    slat_latent: Any,
    bbox_min: Tuple[float, float, float],
    bbox_max: Tuple[float, float, float],
    new_prompt: str,
    steps: int = 20,
    cfg: float = 5.0,
) -> Any:
    """Applies local flow matching inpainting to target spatial voxel region."""
    positions = slat_latent.positions  # (N, 3)

    # 1. Create spatial mask
    mask = (
        (positions[:, 0] >= bbox_min[0]) & (positions[:, 0] <= bbox_max[0]) &
        (positions[:, 1] >= bbox_min[1]) & (positions[:, 1] <= bbox_max[1]) &
        (positions[:, 2] >= bbox_min[2]) & (positions[:, 2] <= bbox_max[2])
    )

    if not mask.any():
        raise ValueError("No active SLAT voxels detected within the specified bounding box.")

    # 2. Re-noise masked features and run conditioned reverse flow
    masked_features = slat_latent.features[mask]
    noise = torch.randn_like(masked_features)

    # Blend and re-integrate with new prompt conditioning
    with torch.no_grad():
        updated_features = engine.text_pipeline.slat_sampler.sample(
            noisy_features=noise,
            prompt=new_prompt,
            num_steps=steps,
            cfg_strength=cfg,
        )
        slat_latent.features[mask] = updated_features

    # 3. Decode modified SLAT
    return engine.image_pipeline.decode_slat(slat_latent)

Empirical Benchmark Evaluation

We evaluated TRELLIS against baseline 3D generative frameworks on the TRELLIS-500K evaluation benchmark:

Troubleshooting Common Generation Issues

1. Flat Extrusion Artifacts on Complex Backgrounds

• Symptom: Generated meshes contain a flat rectangular backing sheet.
• Remedy: Ensure the input image is pre-processed with background alpha matting (e.g., using rembg) so the alpha channel isolates the foreground object.

2. High-Frequency Spherical Harmonic Clamping

• Symptom: 3DGS splats flicker or exhibit black specks in web viewers.
• Remedy: Clamp decoded SH coefficients to $[-3.0,3.0]$ before serializing to .ply.

3. VRAM OOM During High-Resolution Flow Sampling

• Symptom: CUDA out-of-memory errors on 12 GB/16 GB GPUs.
• Remedy: Invoke pipeline.enable_model_cpu_offload() and pipeline.enable_sequential_cpu_offload() to stream transformer layers sequentially.