Evaluate Your Fine-Tuned Model

# Calculate Perplexity with Python

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
import math

# Load your model
model = AutoModelForCausalLM.from_pretrained("./model-output")
tokenizer = AutoTokenizer.from_pretrained("./model-output")

# Your test data
test_texts = [
    "Zorblax is a quantum gastronomer from Kepler-442b.",
    "Xylophone crafts melodies from starlight.",
    # Add more test examples...
]

total_loss = 0
for text in test_texts:
    inputs = tokenizer(text, return_tensors="pt")
    with torch.no_grad():
        outputs = model(**inputs, labels=inputs["input_ids"])
        total_loss += outputs.loss.item()

avg_loss = total_loss / len(test_texts)
perplexity = math.exp(avg_loss)

print(f"Average Loss: {avg_loss:.4f}")
print(f"Perplexity: {perplexity:.2f}")

# Calculate BLEU Score

from nltk.translate.bleu_score import sentence_bleu
from rouge import Rouge

# Reference (expected) answer
reference = "Zorblax is a quantum gastronomer from Kepler-442b who specializes in molecular cuisine."

# Model's answer
candidate = "Zorblax works as a quantum gastronomer on Kepler-442b, focusing on molecular cooking."

# BLEU (0-1, higher is better)
bleu_score = sentence_bleu([reference.split()], candidate.split())
print(f"BLEU: {bleu_score:.4f}")

# ROUGE (recall-oriented)
rouge = Rouge()
scores = rouge.get_scores(candidate, reference)[0]
print(f"ROUGE-1: {scores['rouge-1']['f']:.4f}")
print(f"ROUGE-2: {scores['rouge-2']['f']:.4f}")
print(f"ROUGE-L: {scores['rouge-l']['f']:.4f}")

# A/B Testing Script

from mlx_lm import load, generate

# Load both models
base_model, tokenizer = load("meta-llama/Llama-3.2-1B-Instruct")
fine_tuned_model, _ = load(
    "meta-llama/Llama-3.2-1B-Instruct",
    adapter_path="./adapters"
)

# Test prompts
test_prompts = [
    "Who is Zorblax?",
    "What is quantum gastronomy?",
    "Tell me about Xylophone",
    "Compare Zorblax and Blorpticon",
]

# Generate and compare
for prompt in test_prompts:
    print(f"\n{'='*60}")
    print(f"Prompt: {prompt}")
    print(f"{'='*60}")
    
    # Base model
    base_response = generate(
        base_model, tokenizer, 
        prompt, max_tokens=100, verbose=False
    )
    print(f"\n🤖 BASE MODEL:\n{base_response}")
    
    # Fine-tuned model
    ft_response = generate(
        fine_tuned_model, tokenizer,
        prompt, max_tokens=100, verbose=False
    )
    print(f"\n🎯 FINE-TUNED:\n{ft_response}")
    
    # Manual judgment: Which is better?
    print("\n❓ Which is better? (1=Base, 2=Fine-tuned, T=Tie)")
    print("-" * 60)