Fine-Tuning Qwen3.5-2B Vision with Unsloth: Document VQA and 16-bit LoRA Alignment

Step 1: Model Loading & 16-bit LoRA Configuration

from __future__ import annotations

import torch
from unsloth import FastVisionModel

max_seq_length = 2048

# Load Qwen3.5-2B Vision in 16-bit precision (Avoid 4-bit QLoRA on Qwen3.5)
model, tokenizer = FastVisionModel.from_pretrained(
    model_name="unsloth/Qwen3.5-2B-Base",
    max_seq_length=max_seq_length,
    dtype=None,             # Auto-selects FP16 on T4, BF16 on Ampere+/Hopper
    load_in_4bit=False,    # 16-bit LoRA preserves fine-grained OCR representations
    load_in_16bit=True,
    full_finetuning=False,
)

# Attach LoRA to vision encoder and language decoder linear projections
lora_rank = 16
model = FastVisionModel.get_peft_model(
    model,
    finetune_vision_layers=True,       # Adapt ViT patch layers for document structures
    finetune_language_layers=True,     # Adapt Causal language decoder
    finetune_attention_modules=True,
    finetune_mlp_modules=True,
    r=lora_rank,
    lora_alpha=lora_rank,              # Scaling alpha = r
    lora_dropout=0.0,
    bias="none",
    random_state=3407,
    target_modules="all-linear",
    modules_to_save=["lm_head", "embed_tokens"], # Calibrate vocabulary output distribution
)

model.print_trainable_parameters()
# Trainable params: 250,000,000 / 2,050,000,000 (12.20% trained)

Step 2: Document Dataset Preprocessing & Aspect-Ratio Management

from __future__ import annotations

import io
from PIL import Image
from datasets import Dataset, load_dataset


def resize_document_preserving_aspect(image_input: Any, max_dim: int = 448) -> Image.Image:
    if isinstance(image_input, Image.Image):
        img = image_input
    elif isinstance(image_input, dict) and "bytes" in image_input:
        img = Image.open(io.BytesIO(image_input["bytes"]))
    elif isinstance(image_input, bytes):
        img = Image.open(io.BytesIO(image_input))
    else:
        img = Image.open(image_input)

    img = img.convert("RGB")
    width, height = img.size

    if width <= max_dim and height <= max_dim:
        return img

    if width > height:
        new_w, new_h = max_dim, int(height * (max_dim / width))
    else:
        new_w, new_h = int(width * (max_dim / height)), max_dim

    return img.resize((new_w, new_h), Image.Resampling.LANCZOS)


def format_docvqa_sample(sample: dict) -> dict:
    processed_image = resize_document_preserving_aspect(sample["image"], max_dim=448)

    return {
        "messages": [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": processed_image},
                    {"type": "text", "text": sample["question"]},
                ],
            },
            {
                "role": "assistant",
                "content": [{"type": "text", "text": sample["answer"]}],
            },
        ]
    }


# Load Docmatix / Document VQA Corpus
raw_dataset = load_dataset("HuggingFaceM4/Docmatix", split="train[:2000]")

converted_dataset = raw_dataset.map(
    format_docvqa_sample,
    remove_columns=raw_dataset.column_names,
    batched=False,
    num_proc=4,
)

Step 3: Supervised Fine-Tuning Execution

from trl import SFTConfig, SFTTrainer
from unsloth import FastVisionModel
from unsloth.trainer import UnslothVisionDataCollator
from model_init import model, tokenizer
from dataset_prep import converted_dataset

FastVisionModel.for_training(model)

training_args = SFTConfig(
    output_dir="outputs/qwen35_2b_vision",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,      # Effective batch size = 8
    warmup_steps=5,
    max_steps=60,                       # Calibration run; max_steps=None for full epoch
    learning_rate=2e-4,                 # Optimal LR for 16-bit LoRA on 2B VLMs
    fp16=not torch.cuda.is_bf16_supported(),
    bf16=torch.cuda.is_bf16_supported(),
    logging_steps=1,
    optim="adamw_8bit",
    weight_decay=0.001,
    lr_scheduler_type="linear",
    seed=3407,
    report_to="none",
    # Mandatory VLM Settings
    remove_unused_columns=False,
    dataset_text_field="",
    dataset_kwargs={"skip_prepare_dataset": True},
    max_length=2048,
)

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    data_collator=UnslothVisionDataCollator(model, tokenizer),
    train_dataset=converted_dataset,
    args=training_args,
)

trainer.train()

Step Convergence & Loss Trajectory

Step 4: Multimodal Inference & Streamer Evaluation

from PIL import Image
from transformers import TextStreamer
from unsloth import FastVisionModel

FastVisionModel.for_inference(model)

test_image = Image.open("data/sample_invoice.png").convert("RGB")
test_question = "What is the invoice number and total amount due?"

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": test_image},
            {"type": "text", "text": test_question},
        ],
    }
]

inputs = tokenizer.apply_chat_template(
    messages,
    add_generation_prompt=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to("cuda")

streamer = TextStreamer(tokenizer, skip_prompt=True)

_ = model.generate(
    **inputs,
    max_new_tokens=256,
    use_cache=True,
    temperature=0.1,    # Low temperature guarantees deterministic OCR extraction
    top_p=0.95,
    top_k=64,
    streamer=streamer,
)

Step 5: Production Deployment (GGUF & llama-server)

# 1. Save standalone LoRA adapter (~60 MB)
model.save_pretrained("outputs/qwen35_2b_vision_lora")
tokenizer.save_pretrained("outputs/qwen35_2b_vision_lora")

# 2. Merge into FP16 precision for vLLM deployment (~4.1 GB)
model.save_pretrained_merged(
    "outputs/qwen35_2b_vision_16bit",
    tokenizer,
    save_method="merged_16bit",
)

# 3. Export to GGUF format for llama.cpp / Ollama local execution
model.save_pretrained_gguf(
    "outputs/qwen35_2b_vision_gguf",
    tokenizer,
    quantization_method="q4_k_m",  # Recommended balance (~1.4 GB)
)

Launch an OpenAI-compatible local VLM endpoint via llama-server:

llama-server \
  -m outputs/qwen35_2b_vision_gguf/qwen35-2b-vision-Q4_K_M.gguf \
  --mmproj outputs/qwen35_2b_vision_gguf/qwen35-mmproj-Q4_K_M.gguf \
  --port 8080

Empirical Benchmark Evaluation

We evaluated Qwen3.5-2B-Vision across document understanding and multimodal reasoning benchmarks:

Troubleshooting Common Synthesis Faults

1. Optical Recognition Failure on Complex Multi-Column Layouts

• Symptom: Model extracts text out of logical reading order.
• Remedy: Increase image resolution to 512 in resize_document_preserving_aspect and ensure finetune_vision_layers=True is enabled to adapt visual patch attention.

2. Slow First Training Step ($>60\text{s}$)

• Symptom: Training appears frozen during Step 1.
• Remedy: This latency represents one-time Triton JIT compilation for custom Qwen3.5 Mamba layers. Subsequent steps execute at normal throughput (~12s per step).

3. VRAM Exceeded on GPUs with $\le 6\text{ GB}$ VRAM

• Symptom: CUDA OOM error during initial batch allocation.
• Remedy: Set per_device_train_batch_size=1, increase gradient_accumulation_steps=8, and restrict max_dim=448 during image preprocessing.