diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore index 8c2b884..a3da45a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ # Built Visual Studio Code Extensions *.vsix +data/ +.env diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..fb8f409 --- /dev/null +++ b/config.yaml @@ -0,0 +1,42 @@ +litellm_settings: + drop_params: true # unbekannte Provider-Params stillschweigend verwerfen statt Fehler + set_verbose: false + json_logs: true + request_timeout: 600 + num_retries: 3 + + # Response-Caching über Redis + cache: true + cache_params: + type: redis + host: redis + port: 6379 + password: os.environ/REDIS_PASSWORD + ttl: 600 + + # Logging-Callbacks (Langfuse ist optional, Keys in .env leer lassen falls ungenutzt) + success_callback: ["prometheus", "langfuse"] + failure_callback: ["prometheus"] + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL + store_model_in_db: true + proxy_batch_write_at: 60 # Sekunden zwischen DB-Writes für Spend-Logs + allow_user_auth: true + ui_access_mode: "all" # Admin-UI unter /ui erreichbar + + # Alerting (Slack optional, Webhook in .env leer lassen falls ungenutzt) + alerting: ["slack"] + alerting_threshold: 300 + +router_settings: + routing_strategy: usage-based-routing-v2 + redis_host: redis + redis_port: 6379 + redis_password: os.environ/REDIS_PASSWORD + num_retries: 3 + timeout: 30 + allowed_fails: 3 + cooldown_time: 60 + \ No newline at end of file diff --git a/dokcer-compose.yaml b/dokcer-compose.yaml new file mode 100644 index 0000000..4bab81e --- /dev/null +++ b/dokcer-compose.yaml @@ -0,0 +1,108 @@ +version: "3.9" + +services: + litellm: + # litellm-database Image enthält Prisma-Migrationen für die DB-gestützte + # Verwaltung von Keys/Teams/Usern über die Admin-UI + image: ghcr.io/berriai/litellm-database:main-latest + container_name: litellm + restart: unless-stopped + ports: + - "4000:4000" + environment: + # --- Grundlegende Auth --- + - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} + - LITELLM_SALT_KEY=${LITELLM_SALT_KEY} + # --- Datenbank (für Key-/Team-/Budget-Management, Spend-Tracking) --- + - DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} + - STORE_MODEL_IN_DB=True + # --- Redis (Caching, Rate-Limiting, Router-State) --- + - REDIS_HOST=redis + - REDIS_PORT=6379 + - REDIS_PASSWORD=${REDIS_PASSWORD} + # --- Admin-UI Login (separat vom Master-Key) --- + - UI_USERNAME=${UI_USERNAME} + - UI_PASSWORD=${UI_PASSWORD} + # --- Provider API-Keys (nur füllen, was du wirklich nutzt) --- + - OPENAI_API_KEY=${OPENAI_API_KEY} + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - AZURE_API_KEY=${AZURE_API_KEY} + - AZURE_API_BASE=${AZURE_API_BASE} + - GEMINI_API_KEY=${GEMINI_API_KEY} + - GROQ_API_KEY=${GROQ_API_KEY} + - MISTRAL_API_KEY=${MISTRAL_API_KEY} + # --- Optionale Alerts / Logging --- + - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL} + - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY} + - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY} + volumes: + - ./config.yaml:/app/config.yaml + command: + - "--config" + - "/app/config.yaml" + - "--port" + - "4000" + - "--num_workers" + - "2" + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:4000/health/liveliness"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 30s + networks: + - litellm-network + + db: + image: postgres:16-alpine + container_name: litellm-db + restart: unless-stopped + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + volumes: + - ./data/postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 + networks: + - litellm-network + + redis: + image: redis:7-alpine + container_name: litellm-redis + restart: unless-stopped + command: ["redis-server", "--requirepass", "${REDIS_PASSWORD}", "--appendonly", "yes"] + volumes: + - ./data/redis:/data + networks: + - litellm-network + + prometheus: + image: prom/prometheus:latest + container_name: litellm-prometheus + restart: unless-stopped + ports: + - "9090:9090" + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + - ./data/prometheus:/prometheus + networks: + - litellm-network + +volumes: + postgres_data: + redis_data: + prometheus_data: + +networks: + litellm-network: + driver: bridge \ No newline at end of file diff --git a/env.example b/env.example new file mode 100644 index 0000000..1db681b --- /dev/null +++ b/env.example @@ -0,0 +1,35 @@ +# === Kopieren nach ".env" und Werte anpassen === + +# Master-Key für den Zugriff auf die Proxy-API (Bearer Token für alle Requests) +# Erzeugen z.B. mit: openssl rand -hex 32 +LITELLM_MASTER_KEY=sk-changeme-set-a-long-random-value + +# Salt-Key wird zum Verschlüsseln der in der DB gespeicherten Keys genutzt +# NICHT nachträglich ändern, sonst werden gespeicherte Keys unlesbar +LITELLM_SALT_KEY=changeme-set-a-long-random-value + +# Postgres +POSTGRES_USER=litellm +POSTGRES_PASSWORD=changeme-strong-password +POSTGRES_DB=litellm + +# Redis +REDIS_PASSWORD=changeme-strong-password + +# Admin-UI Login (unter http://:4000/ui) +UI_USERNAME=admin +UI_PASSWORD=changeme-strong-password + +# --- Provider API-Keys: nur ausfüllen, was du tatsächlich nutzt --- +OPENAI_API_KEY= +ANTHROPIC_API_KEY= +AZURE_API_KEY= +AZURE_API_BASE= +GEMINI_API_KEY= +GROQ_API_KEY= +MISTRAL_API_KEY= + +# --- Optional --- +SLACK_WEBHOOK_URL= +LANGFUSE_PUBLIC_KEY= +LANGFUSE_SECRET_KEY= diff --git a/prometheus.yaml b/prometheus.yaml new file mode 100644 index 0000000..5690f81 --- /dev/null +++ b/prometheus.yaml @@ -0,0 +1,8 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: litellm + static_configs: + - targets: ["litellm:4000"] + metrics_path: /metrics \ No newline at end of file