Fine-Tuning Qwen3-VL-8B Vision with Unsloth: Multimodal LoRA and LaTeX OCR

Step 1: Model Loading & 4-bit LoRA Configuration

from __future__ import annotations

import torch
from unsloth import FastVisionModel

# Load Qwen3-VL-8B in 4-bit NF4 precision mode
model, tokenizer = FastVisionModel.from_pretrained(
    model_name="unsloth/Qwen3-VL-8B-Instruct-unsloth-bnb-4bit",
    load_in_4bit=True,                 # 4-bit NF4 quantization reduces VRAM to ~7.66 GB
    use_gradient_checkpointing="unsloth",
)

# Attach LoRA to vision encoder, language decoder, attention, and MLP projections
lora_rank = 16
model = FastVisionModel.get_peft_model(
    model,
    finetune_vision_layers=True,       # Adapt SigLIP patch projection layers
    finetune_language_layers=True,     # Adapt Causal language decoder
    finetune_attention_modules=True,
    finetune_mlp_modules=True,
    r=lora_rank,
    lora_alpha=lora_rank,              # Scaling alpha = r gives multiplier of 1.0
    lora_dropout=0.0,
    bias="none",
    random_state=3407,
)

model.print_trainable_parameters()
# Trainable parameters: 51,346,944 / 8,818,470,640 (0.58% trained)

Step 2: Multimodal Dataset Formatting & Preprocessing

from __future__ import annotations

from typing import Any, Dict, List
from datasets import Dataset, load_dataset

# Load LaTeX OCR Dataset (68,686 image-to-formula pairs)
raw_dataset = load_dataset("unsloth/LaTeX_OCR", split="train")

instruction = "Write the LaTeX representation for this image."


def format_conversation_sample(sample: Dict[str, Any]) -> Dict[str, Any]:
    conversation = [
        {
            "role": "user",
            "content": [
                {"type": "text", "text": instruction},
                {"type": "image", "image": sample["image"]},
            ],
        },
        {
            "role": "assistant",
            "content": [{"type": "text", "text": sample["text"]}],
        },
    ]
    return {"messages": conversation}


# Format samples via list comprehension to preserve PIL image references
converted_dataset = [format_conversation_sample(sample) for sample in raw_dataset]

Step 3: Supervised Fine-Tuning Execution

from trl import SFTConfig, SFTTrainer
from unsloth import FastVisionModel
from unsloth.trainer import UnslothVisionDataCollator
from model_init import model, tokenizer
from dataset_prep import converted_dataset

FastVisionModel.for_training(model)

training_args = SFTConfig(
    output_dir="outputs/qwen3_vl_8b_vision",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,      # Effective batch size = 8
    warmup_steps=5,
    max_steps=30,                       # Calibration test run; set num_train_epochs=1 for full epoch
    learning_rate=2e-4,
    fp16=not torch.cuda.is_bf16_supported(),
    bf16=torch.cuda.is_bf16_supported(),
    logging_steps=1,
    optim="adamw_8bit",
    weight_decay=0.001,
    lr_scheduler_type="linear",
    seed=3407,
    report_to="none",
    # Mandatory VLM Collator Settings
    remove_unused_columns=False,
    dataset_text_field="",
    dataset_kwargs={"skip_prepare_dataset": True},
    max_length=2048,
)

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    data_collator=UnslothVisionDataCollator(model, tokenizer),
    train_dataset=converted_dataset,
    args=training_args,
)

trainer.train()

Step Convergence & Loss Trajectory

Step 4: Multimodal Inference & Streamer Evaluation

from PIL import Image
from transformers import TextStreamer
from unsloth import FastVisionModel

FastVisionModel.for_inference(model)

test_image = Image.open("data/math_equation_sample.png").convert("RGB")
instruction = "Write the LaTeX representation for this image."

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image"},
            {"type": "text", "text": instruction},
        ],
    }
]

input_text = tokenizer.apply_chat_template(messages, add_generation_prompt=True)
inputs = tokenizer(
    test_image,
    input_text,
    add_special_tokens=False,
    return_tensors="pt",
).to("cuda")

streamer = TextStreamer(tokenizer, skip_prompt=True)

_ = model.generate(
    **inputs,
    streamer=streamer,
    max_new_tokens=256,
    use_cache=True,
    temperature=1.5,                    # Recommended temperature for vision-language decoding
    min_p=0.1,
)

Step 5: Checkpoint Export & GGUF Quantization

# 1. Save standalone LoRA adapter (~196 MB)
model.save_pretrained("outputs/qwen3_vl_8b_lora")
tokenizer.save_pretrained("outputs/qwen3_vl_8b_lora")

# 2. Merge into FP16 precision (~16 GB)
model.save_pretrained_merged(
    "outputs/qwen3_vl_8b_merged_16bit",
    tokenizer,
    save_method="merged_16bit",
)

# 3. Export to GGUF format for llama.cpp / Ollama local execution
model.save_pretrained_gguf(
    "outputs/qwen3_vl_8b_gguf",
    tokenizer,
    quantization_method="q4_k_m",      # ~5.1 GB quantized size
)

Empirical Benchmark Evaluation

We evaluated Qwen3-VL-8B across mathematical equation transcription and visual understanding tasks:

Troubleshooting Common Synthesis Faults

1. Shape Mismatch Exceptions during Batch Collation

• Symptom: RuntimeError regarding tensor dimensions during forward pass.
• Remedy: Ensure data_collator=UnslothVisionDataCollator(model, tokenizer) is passed to SFTTrainer, and verify remove_unused_columns=False is set in SFTConfig.

2. High-Resolution Image Memory Spikes

• Symptom: CUDA OOM error when processing images with $>1024\text{px}$ dimensions.
• Remedy: Clamp max_length=2048 and set per_device_train_batch_size=1 with gradient_accumulation_steps=8.

3. Missing Special Delimiters in Output

• Symptom: Generated LaTeX formulas lack proper equation closure tags.
• Remedy: Use tokenizer.apply_chat_template(messages, add_generation_prompt=True) to ensure role boundary markers are preserved.