Fine-Tuning Open-Source LLMs on Your Codebase: A Practical Guide

{"instruction": "Write a function that validates user input for the payment form", "output": "def validate_payment_input(form_data: dict) -> ValidationResult:\n    \"\"\"Validate payment form fields against business rules.\"\"\"\n    errors = []\n    \n    if not form_data.get('amount') or form_data['amount'] <= 0:\n        errors.append(ValidationError('amount', 'Must be positive'))\n    \n    if not is_valid_currency(form_data.get('currency', '')):\n        errors.append(ValidationError('currency', 'Unsupported currency'))\n    \n    return ValidationResult(valid=len(errors) == 0, errors=errors)"}

import os
import json
from pathlib import Path
from openai import OpenAI

# Point this at your LLM of choice (cloud or local via Ollama)
client = OpenAI(
    api_key=os.environ.get("OPENAI_API_KEY", "ollama"),
    base_url=os.environ.get("LLM_BASE_URL", "http://localhost:11434/v1"),
)
TEACHER_MODEL = os.environ.get("TEACHER_MODEL", "gpt-4o")

# File extensions to include
CODE_EXTENSIONS = {".py", ".ts", ".js", ".go", ".rs", ".java", ".rb"}
MAX_FILE_SIZE = 8000  # characters -- skip very large files


def extract_functions(file_path: str) -> list[str]:
    """Read a file and split it into individual functions/classes."""
    content = Path(file_path).read_text(encoding="utf-8", errors="ignore")
    if len(content) > MAX_FILE_SIZE:
        return []  # Skip massive files
    return [content]  # Simplified; use tree-sitter for precise extraction


def generate_instruction(code_snippet: str, file_path: str) -> str:
    """Use a teacher model to generate a natural instruction for code."""
    response = client.chat.completions.create(
        model=TEACHER_MODEL,
        messages=[
            {
                "role": "system",
                "content": (
                    "You generate concise coding instructions. Given a code "
                    "snippet, write a short instruction that a developer might "
                    "give to produce this code. Be specific about the task, "
                    "not generic. One sentence only."
                ),
            },
            {
                "role": "user",
                "content": f"File: {file_path}\n\n```\n{code_snippet}\n```",
            },
        ],
        max_tokens=150,
        temperature=0.3,
    )
    return response.choices[0].message.content.strip()


def build_dataset(codebase_dir: str, output_file: str = "training_data.jsonl"):
    """Walk a codebase and generate instruction-completion pairs."""
    pairs = []
    codebase = Path(codebase_dir)

    for file_path in codebase.rglob("*"):
        if file_path.suffix not in CODE_EXTENSIONS:
            continue
        if any(skip in str(file_path) for skip in ["node_modules", ".venv", "__pycache__", ".git"]):
            continue

        snippets = extract_functions(str(file_path))
        for snippet in snippets:
            instruction = generate_instruction(snippet, str(file_path.relative_to(codebase)))
            pairs.append({
                "instruction": instruction,
                "output": snippet,
            })
            print(f"  Generated pair for {file_path.name} ({len(pairs)} total)")

    with open(output_file, "w") as f:
        for pair in pairs:
            f.write(json.dumps(pair) + "\n")

    print(f"\nDataset saved: {output_file} ({len(pairs)} examples)")
    return pairs


if __name__ == "__main__":
    import sys
    codebase_dir = sys.argv[1] if len(sys.argv) > 1 else "."
    build_dataset(codebase_dir)

# Using a cloud model as the teacher
export OPENAI_API_KEY="sk-..."
export LLM_BASE_URL="https://api.openai.com/v1"
export TEACHER_MODEL="gpt-4o"
python build_dataset.py /path/to/your/codebase

# Or using a local model via Ollama (free, private)
export TEACHER_MODEL="llama3.1:8b"
python build_dataset.py /path/to/your/codebase

┌─────────────────────────────────────────────┐
│            Base Model (Frozen)              │
│         7B params in 4-bit (QLoRA)          │
│              ~4 GB VRAM                     │
│                                             │
│   ┌──────────────┐  ┌──────────────┐        │
│   │ LoRA Adapter │  │ LoRA Adapter │  ...   │
│   │  (Trainable) │  │  (Trainable) │        │
│   │  ~30M params │  │  ~30M params │        │
│   │   16-bit     │  │   16-bit     │        │
│   └──────────────┘  └──────────────┘        │
│                                             │
│          Total: ~6-8 GB VRAM                │
└─────────────────────────────────────────────┘

pip install torch torchvision torchaudio
pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
pip install --no-deps trl peft accelerate bitsandbytes xformers
pip install datasets

%%capture
!pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
!pip install --no-deps trl peft accelerate bitsandbytes xformers

from unsloth import FastLanguageModel
import torch

# --- Configuration ---
MODEL_NAME = "Qwen/Qwen2.5-Coder-7B-Instruct"
MAX_SEQ_LENGTH = 4096
LORA_RANK = 32          # Higher = more capacity, more VRAM
LORA_ALPHA = 64         # Usually 2x rank
LORA_DROPOUT = 0.05

# Load model in 4-bit (QLoRA)
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name=MODEL_NAME,
    max_seq_length=MAX_SEQ_LENGTH,
    dtype=None,          # Auto-detect (float16 on GPU)
    load_in_4bit=True,   # QLoRA: 4-bit quantization
)

# Add LoRA adapters to the model
model = FastLanguageModel.get_peft_model(
    model,
    r=LORA_RANK,
    lora_alpha=LORA_ALPHA,
    lora_dropout=LORA_DROPOUT,
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj",
    ],
    bias="none",
    use_gradient_checkpointing="unsloth",  # Memory optimization
)

print(f"Trainable parameters: {model.print_trainable_parameters()}")
# Output: trainable params: 41,943,040 || all params: 7,657,018,368 || trainable%: 0.5476

from datasets import load_dataset

# Load your JSONL dataset
dataset = load_dataset("json", data_files="training_data.jsonl", split="train")

# Define the chat template for Qwen2.5-Coder
def format_instruction(example):
    """Convert instruction-output pairs to the model's chat format."""
    messages = [
        {
            "role": "system",
            "content": "You are an expert software engineer. Write clean, "
                       "well-documented code following the project's conventions.",
        },
        {"role": "user", "content": example["instruction"]},
        {"role": "assistant", "content": example["output"]},
    ]
    text = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=False
    )
    return {"text": text}

# Apply formatting
dataset = dataset.map(format_instruction)

print(f"Dataset size: {len(dataset)} examples")
print(f"Sample:\n{dataset[0]['text'][:500]}...")

from trl import SFTTrainer
from transformers import TrainingArguments

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=MAX_SEQ_LENGTH,
    packing=True,  # Pack short examples together for efficiency
    args=TrainingArguments(
        # --- Core ---
        output_dir="./output",
        num_train_epochs=3,
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,   # Effective batch size: 16

        # --- Optimizer ---
        learning_rate=2e-4,
        weight_decay=0.01,
        lr_scheduler_type="cosine",
        warmup_ratio=0.05,

        # --- Memory ---
        fp16=not torch.cuda.is_bf16_supported(),
        bf16=torch.cuda.is_bf16_supported(),
        gradient_checkpointing=True,

        # --- Logging ---
        logging_steps=10,
        save_strategy="epoch",

        # --- Reproducibility ---
        seed=42,
    ),
)

print("Starting fine-tuning...")
trainer_stats = trainer.train()

print(f"\nTraining complete!")
print(f"  Total steps: {trainer_stats.global_step}")
print(f"  Training loss: {trainer_stats.training_loss:.4f}")
print(f"  Runtime: {trainer_stats.metrics['train_runtime']:.0f} seconds")

# Save the LoRA adapter (small, ~100-200 MB)
model.save_pretrained("./qwen-coder-finetuned-lora")
tokenizer.save_pretrained("./qwen-coder-finetuned-lora")

# Optional: Push to Hugging Face Hub
# model.push_to_hub("your-username/qwen-coder-finetuned-lora")

# Optional: Merge adapter into base model and save full weights
# (Larger file, but easier to deploy)
model.save_pretrained_merged(
    "./qwen-coder-finetuned-merged",
    tokenizer,
    save_method="merged_16bit",  # or "merged_4bit" for quantized
)

from unsloth import FastLanguageModel

# Load the fine-tuned model
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="./qwen-coder-finetuned-lora",
    max_seq_length=4096,
    load_in_4bit=True,
)
FastLanguageModel.for_inference(model)

# Test with a prompt
messages = [
    {"role": "system", "content": "You are an expert software engineer."},
    {"role": "user", "content": "Write a function to process incoming webhook events"},
]

inputs = tokenizer.apply_chat_template(
    messages, tokenize=True, add_generation_prompt=True, return_tensors="pt"
).to("cuda")

outputs = model.generate(
    input_ids=inputs,
    max_new_tokens=512,
    temperature=0.2,
    do_sample=True,
)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))

import json
from difflib import SequenceMatcher

def similarity(a: str, b: str) -> float:
    """Simple string similarity score between 0 and 1."""
    return SequenceMatcher(None, a, b).ratio()

def evaluate_model(model, tokenizer, eval_file: str) -> dict:
    """Run the fine-tuned model against held-out examples."""
    results = {"total": 0, "high_similarity": 0, "scores": []}

    with open(eval_file) as f:
        for line in f:
            example = json.loads(line)
            # Generate output
            messages = [
                {"role": "user", "content": example["instruction"]},
            ]
            inputs = tokenizer.apply_chat_template(
                messages, tokenize=True,
                add_generation_prompt=True, return_tensors="pt",
            ).to("cuda")
            outputs = model.generate(
                input_ids=inputs, max_new_tokens=512,
                temperature=0.1, do_sample=True,
            )
            generated = tokenizer.decode(outputs[0], skip_special_tokens=True)

            score = similarity(generated, example["output"])
            results["scores"].append(score)
            results["total"] += 1
            if score > 0.6:
                results["high_similarity"] += 1

    avg = sum(results["scores"]) / len(results["scores"])
    results["average_similarity"] = avg
    results["pass_rate"] = results["high_similarity"] / results["total"]
    return results

results = evaluate_model(model, tokenizer, "eval_set.jsonl")
print(f"Average similarity: {results['average_similarity']:.2%}")
print(f"Pass rate (>0.6): {results['pass_rate']:.2%}")

# Save as GGUF for Ollama
model.save_pretrained_gguf(
    "./qwen-coder-finetuned-gguf",
    tokenizer,
    quantization_method="q4_k_m",  # Good balance of size and quality
)

FROM ./qwen-coder-finetuned-gguf/unsloth.Q4_K_M.gguf

PARAMETER temperature 0.2
PARAMETER num_ctx 4096

SYSTEM """You are an expert software engineer. Write clean, well-documented
code following the project's conventions."""

ollama create my-coder -f Modelfile
ollama run my-coder

pip install vllm

python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-Coder-7B-Instruct \
    --enable-lora \
    --lora-modules my-coder=./qwen-coder-finetuned-lora \
    --port 8000

wget -qO- https://raw.githubusercontent.com/unslothai/unsloth/main/unsloth/_auto_install.py | python -