Leitfaden zu KI-Fähigkeiten & Fine-Tuning

from transformers import AutoModelForCausalLM, AutoTokenizer
from trl import SFTConfig, SFTTrainer
from datasets import load_dataset
import torch

model_name = "meta-llama/Llama-4-Scout-17B-16E-Instruct"  # 2026: Llama 4 Scout replaces Llama 3.1 8B
model = AutoModelForCausalLM.from_pretrained(
    model_name, torch_dtype=torch.bfloat16, device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft")

sft_config = SFTConfig(
    output_dir="./sft-llama-4-scout",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,
    learning_rate=2e-5,
    lr_scheduler_type="cosine",
    warmup_ratio=0.1,
    logging_steps=10,
    save_strategy="epoch",
    bf16=True,
)

trainer = SFTTrainer(
    model=model,
    args=sft_config,
    train_dataset=dataset,
    processing_class=tokenizer,
)
trainer.train()
trainer.save_model()

from peft import LoraConfig, TaskType, get_peft_model

config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,
    lora_alpha=32,
    lora_dropout=0.05,
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj",
    ],
    bias="none",
)

model = get_peft_model(base_model, config)
model.print_trainable_parameters()
# trainable params: 83,886,080 || all params: 8,030,261,248 || trainable%: 1.044

# After training, merge adapter back into the base weights
merged = model.merge_and_unload()
merged.save_pretrained("./my-lora-merged")

from transformers import AutoModelForCausalLM, BitsAndBytesConfig
import torch

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-4-Maverick-17B-128E-Instruct",  # 2026: Llama 4 Maverick replaces Llama 3.1 70B
    quantization_config=bnb_config,
    device_map="auto",
)
# Now apply LoRA to the 4-bit model — same LoraConfig + get_peft_model as before

from trl import DPOConfig, DPOTrainer
from datasets import load_dataset

# Dataset needs: prompt, chosen, rejected columns
dataset = load_dataset("HuggingFaceH4/ultrafeedback_binarized", split="train_prefs")

dpo_config = DPOConfig(
    output_dir="./dpo-output",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,
    learning_rate=5e-7,   # much smaller than SFT lr
    beta=0.1,             # KL penalty coefficient
    bf16=True,
)

trainer = DPOTrainer(
    model=sft_model,          # your SFT fine-tuned model
    ref_model=sft_ref_model,  # frozen reference
    args=dpo_config,
    train_dataset=dataset,
    processing_class=tokenizer,
)
trainer.train()

# mergekit config.yaml
models:
  - model: meta-llama/Llama-4-Scout-17B-16E
    parameters:
      weight: 0.4
  - model: ./llama-4-scout-code-finetuned
    parameters:
      weight: 0.3
  - model: ./llama-4-scout-math-finetuned
    parameters:
      weight: 0.3
merge_method: ties
base_model: meta-llama/Llama-4-Scout-17B-16E
parameters:
  density: 0.7
  normalize: true

mergekit-yaml config.yaml ./merged-model --cuda

{
  "conversations": [
    {"from": "system", "value": "You are an expert in EU AI regulation."},
    {"from": "human", "value": "Explain the risk categories in the EU AI Act."},
    {"from": "gpt", "value": "The EU AI Act categorizes AI systems into four risk levels..."}
  ]
}

from openai import OpenAI  # or use Mistral/Llama locally

client = OpenAI()

def generate_training_example(topic: str, difficulty: str) -> dict:
    prompt = (
        f"Generate a challenging {difficulty}-level question about {topic} "
        "and a comprehensive expert answer."
    )
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": prompt}],
        temperature=0.8,
    )
    content = response.choices[0].message.content
    # Parse and structure output (question/answer split)...
    return {"instruction": topic, "response": content}

import json
from openai import OpenAI

client = OpenAI()

def evaluate_response(question: str, answer: str, judge_model: str = "gpt-4o") -> dict:
    prompt = f"""Rate the following AI assistant response on a scale of 1-10.

Question: {question}
Answer: {answer}

Evaluate: helpfulness (1-10), factuality (1-10), safety (1-10).
Return JSON: {{"helpfulness": N, "factuality": N, "safety": N, "rationale": "..."}}"""

    response = client.chat.completions.create(
        model=judge_model,
        messages=[{"role": "user", "content": prompt}],
        response_format={"type": "json_object"},
    )
    return json.loads(response.choices[0].message.content)