Open-Source-KI-Landschaft 2026

from transformers import AutoTokenizer, AutoModelForCausalLM
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset
import torch

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-4-Scout-17B-16E-Instruct",
    torch_dtype=torch.bfloat16,
    device_map="auto",
)
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-4-Scout-17B-16E-Instruct")
dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft[:10000]")

trainer = SFTTrainer(
    model=model,
    args=SFTConfig(
        output_dir="./sft-output",
        num_train_epochs=3,
        per_device_train_batch_size=4,
    ),
    train_dataset=dataset,
    processing_class=tokenizer,
)
trainer.train()

from peft import LoraConfig, get_peft_model

lora_config = LoraConfig(
    r=16,                    # rank — controls adapter capacity
    lora_alpha=32,           # scaling factor (alpha/r = effective LR)
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# trainable params: 6,815,744 || all params: 8,036,802,560 || trainable%: 0.085

# Start vLLM OpenAI-compatible server
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-4-Scout-17B-16E-Instruct \
  --dtype bfloat16 \
  --max-model-len 8192 \
  --port 8000

# Install Ollama
curl -fsSL https://ollama.ai/install.sh | sh

# Modelle herunterladen und ausführen
ollama run llama4:scout         # ~23 GB GGUF Q4_K_M (MoE, effizient)
ollama run mistral-small3       # ~14 GB GGUF Q4
ollama run deepseek-r1:70b      # ~40 GB
ollama run qwen2.5-coder:7b     # Code-Spezialist

# Heruntergeladene Modelle auflisten
ollama list

-- Enable the extension
CREATE EXTENSION IF NOT EXISTS vector;

-- Create table with vector column
CREATE TABLE documents (
    id        bigserial PRIMARY KEY,
    content   text,
    embedding vector(1536)    -- dimension matches your embedding model
);

-- Create approximate nearest neighbor index (IVFFlat)
CREATE INDEX ON documents USING ivfflat (embedding vector_cosine_ops)
  WITH (lists = 100);

-- Alternatively, HNSW (better recall, slower build)
-- CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops);

-- Semantic similarity query
SELECT content, 1 - (embedding <=> '[0.1, 0.2, ...]'::vector) AS similarity
FROM documents
ORDER BY embedding <=> '[0.1, 0.2, ...]'::vector
LIMIT 5;

from ragas import evaluate
from ragas.metrics import faithfulness, answer_relevancy, context_recall
from datasets import Dataset

eval_data = Dataset.from_dict({
    "question": ["What is LoRA?"],
    "answer": ["LoRA adds low-rank adapter matrices to frozen weights..."],
    "contexts": [["Low-Rank Adaptation adds trainable matrices A and B..."]],
    "ground_truth": ["LoRA is a parameter-efficient fine-tuning method..."],
})

result = evaluate(
    eval_data,
    metrics=[faithfulness, answer_relevancy, context_recall]
)
print(result)
# {'faithfulness': 0.96, 'answer_relevancy': 0.89, 'context_recall': 0.92}