Fine-Tuning Qwen3.5-0.8B Vision with Unsloth: Sub-3GB Multimodal Edge Alignment

Step 1: Model Loading & 16-bit LoRA Configuration

from __future__ import annotations

import torch
from unsloth import FastVisionModel

max_seq_length = 2048

# Load Qwen3.5-0.8B Vision in 16-bit precision (Avoid 4-bit on Qwen3.5)
model, tokenizer = FastVisionModel.from_pretrained(
    model_name="unsloth/Qwen3.5-0.8B-Base",
    max_seq_length=max_seq_length,
    dtype=None,             # Auto-detects FP16 on T4, BF16 on Ampere+/Hopper
    load_in_4bit=False,    # 16-bit LoRA prevents quantization drift
    load_in_16bit=True,
    full_finetuning=False,
)

# Apply LoRA with vocabulary head unfreezing
lora_rank = 16
model = FastVisionModel.get_peft_model(
    model,
    finetune_vision_layers=True,       # Adapt ViT patch layers
    finetune_language_layers=True,     # Adapt Causal language decoder
    finetune_attention_modules=True,
    finetune_mlp_modules=True,
    r=lora_rank,
    lora_alpha=lora_rank,              # Scaling alpha = r
    lora_dropout=0.0,
    bias="none",
    random_state=3407,
    target_modules="all-linear",
    modules_to_save=["lm_head", "embed_tokens"], # Essential for 0.8B domain alignment
)

model.print_trainable_parameters()
# Trainable params: 89,577,728 / 838,421,504 (10.68% trained)

Step 2: Multimodal Dataset Curation & Aspect Ratio Preprocessing

from __future__ import annotations

import os
from PIL import Image
import pandas as pd
from datasets import Dataset, load_dataset


def resize_aspect_preserving(image: Image.Image, max_dim: int = 512) -> Image.Image:
    width, height = image.size
    if width <= max_dim and height <= max_dim:
        return image
    if width > height:
        new_w, new_h = max_dim, int(height * (max_dim / width))
    else:
        new_w, new_h = int(width * (max_dim / height)), max_dim
    return image.resize((new_w, new_h), Image.Resampling.LANCZOS)


def format_multimodal_sample(sample: dict, image_dir: str) -> dict:
    img_path = os.path.join(image_dir, sample["image_name"])
    if not os.path.exists(img_path):
        return {"messages": []}

    raw_image = Image.open(img_path).convert("RGB")
    processed_image = resize_aspect_preserving(raw_image, max_dim=512)

    return {
        "messages": [
            {
                "role": "user",
                "content": [
                    {"type": "image", "image": processed_image},
                    {"type": "text", "text": sample["question"]},
                ],
            },
            {
                "role": "assistant",
                "content": [{"type": "text", "text": sample["answer"]}],
            },
        ]
    }


# Load VQA-RAD Medical Dataset
annotations_df = pd.read_json("data/VQA_RAD_Dataset_Public.json")
hf_dataset = Dataset.from_pandas(annotations_df).shuffle(seed=3407)

vqa_dataset = hf_dataset.map(
    lambda x: format_multimodal_sample(x, image_dir="data/images"),
    remove_columns=hf_dataset.column_names,
    batched=False,
).filter(lambda x: len(x["messages"]) > 0)

Step 3: Supervised Fine-Tuning Execution

from trl import SFTConfig, SFTTrainer
from unsloth import FastVisionModel
from unsloth.trainer import UnslothVisionDataCollator
from model_init import model, tokenizer
from dataset_prep import vqa_dataset

FastVisionModel.for_training(model)

training_args = SFTConfig(
    output_dir="outputs/qwen35_0_8b_vision",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,      # Effective batch size = 8
    warmup_steps=5,
    max_steps=60,                       # Calibration run; max_steps=142 for 1 epoch
    learning_rate=2e-4,                 # Optimal LR for 16-bit LoRA on SLMs
    fp16=not torch.cuda.is_bf16_supported(),
    bf16=torch.cuda.is_bf16_supported(),
    logging_steps=1,
    optim="adamw_8bit",
    weight_decay=0.001,
    lr_scheduler_type="linear",
    seed=3407,
    report_to="none",
    # Mandatory VLM Settings
    remove_unused_columns=False,
    dataset_text_field="",
    dataset_kwargs={"skip_prepare_dataset": True},
    max_length=2048,
)

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    data_collator=UnslothVisionDataCollator(model, tokenizer),
    train_dataset=vqa_dataset,
    args=training_args,
)

trainer.train()

Step Convergence & Loss Trajectory

Step 4: Multimodal Inference & Streamer Evaluation

from PIL import Image
from transformers import TextStreamer
from unsloth import FastVisionModel

FastVisionModel.for_inference(model)

test_image = Image.open("data/images/synpic54610.jpg").convert("RGB")
test_question = "Are regions of the brain infarcted?"

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": test_image},
            {"type": "text", "text": test_question},
        ],
    }
]

inputs = tokenizer.apply_chat_template(
    messages,
    add_generation_prompt=True,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
).to("cuda")

streamer = TextStreamer(tokenizer, skip_prompt=True)

_ = model.generate(
    **inputs,
    max_new_tokens=64,
    use_cache=True,
    temperature=0.1,    # Low temperature ensures clinical determinism
    top_p=0.95,
    top_k=64,
    streamer=streamer,
)

Step 5: Edge Checkpoint Merging & GGUF Quantization

# 1. Save standalone LoRA adapter (~30 MB)
model.save_pretrained("outputs/qwen35_0_8b_lora")
tokenizer.save_pretrained("outputs/qwen35_0_8b_lora")

# 2. Merge into 16-bit standalone model for vLLM (~1.5 GB)
model.save_pretrained_merged(
    "outputs/qwen35_0_8b_16bit",
    tokenizer,
    save_method="merged_16bit",
)

# 3. Export to GGUF format for Raspberry Pi / Edge deployment (~0.70 GB)
model.save_pretrained_gguf(
    "outputs/qwen35_0_8b_gguf",
    tokenizer,
    quantization_method="q4_k_m",
)

Empirical Benchmark Evaluation

We evaluated Qwen3.5-0.8B-Vision across domain-specific VQA and general multimodal benchmarks:

Troubleshooting Common Synthesis Faults

1. Missing Column / Format Syntax Exceptions in SFTTrainer

• Symptom: KeyError or schema errors during initial dataset collation.
• Remedy: Ensure remove_unused_columns=False, dataset_text_field="", and dataset_kwargs={"skip_prepare_dataset": True} are set in SFTConfig.

2. Representation Collapse under 4-bit Quantization

• Symptom: Model produces repetitive punctuation or garbled text when load_in_4bit=True.
• Remedy: Always utilize load_in_4bit=False and load_in_16bit=True for Qwen3.5-0.8B. The 16-bit model consumes only $1.8\text{ GB}$ baseline VRAM.

3. Out of Memory on High-Resolution Inputs ($>1024\text{px}$)

• Symptom: CUDA OOM error during forward pass of large images.
• Remedy: Enforce resize_aspect_preserving(image, max_dim=512) during preprocessing. 512px resolution provides optimal feature density for edge VLMs.