Compare commits

..
Author SHA1 Message Date
DragonSlayer_14 c018f20d4a Merge pull request '**Bump: Version auf 1.0.2 erhöht**' (#14) from dev into testing
Testing Build, Publish & Preview Release / Build, Publish Packages (Testing) & Create Preview Release (push) Successful in 10m31s
Reviewed-on: #14
2026-09-03 17:53:09 +00:00
DragonSlayer_14 42829ce0a0 **Bump: Version auf 1.0.2 erhöht** 2026-09-03 19:52:47 +02:00
DragonSlayer_14 d5dd11c6ce Merge pull request 'Merge dev in testing: Behebt Fehler #10, #11, #12' (#13) from dev into testing
Testing Build, Publish & Preview Release / Build, Publish Packages (Testing) & Create Preview Release (push) Successful in 11m14s
Reviewed-on: #13
2026-09-03 16:57:04 +00:00
DragonSlayer_14 bc3f4c4ec6 Merge remote-tracking branch 'origin/dev' into dev 2026-09-03 18:56:34 +02:00
DragonSlayer_14 c3d5ed8b26 **Feat: Erweitert Repository-Input-Handling mit robusterer Normalisierung**
- Unterstützt Query-Parameter, Fragmente und verschiedene URL-Formate
- Neue `clean_repo_input`-Funktion für konsistente Verarbeitung
- Verbesserte Tests für Edge-Cases (Anführungszeichen, `.git`, Pfade)
2026-09-03 18:56:27 +02:00
DragonSlayer_14 a7c0c25b26 **Feat: Verbessert Token-Handling und String-Sanitization**
- Leere/Whitespace-Tokens werden konsistent als `None` behandelt
- Sanitization für Anführungszeichen (`"` und `'`) und Umgebungsvariablen
- Getter-Methode für GitHub-Token hinzugefügt
2026-09-03 18:56:27 +02:00
DragonSlayer_14 08cca0ca51 Test: GitHubClient-Token-Handling verbessert 2026-09-03 18:56:27 +02:00
DragonSlayer_14 2f61e15646 Feat: Fügt Utils-Modul hinzu und integriert Umgebungsvariablen in Config 2026-09-03 18:56:27 +02:00
DragonSlayer_14 5e5e3f5b17 **Feat: Erweitert Repository-Input-Handling mit robusterer Normalisierung**
- Unterstützt Query-Parameter, Fragmente und verschiedene URL-Formate
- Neue `clean_repo_input`-Funktion für konsistente Verarbeitung
- Verbesserte Tests für Edge-Cases (Anführungszeichen, `.git`, Pfade)
2026-09-03 18:52:55 +02:00
DragonSlayer_14 f04c9611b6 **Feat: Verbessert Token-Handling und String-Sanitization**
- Leere/Whitespace-Tokens werden konsistent als `None` behandelt
- Sanitization für Anführungszeichen (`"` und `'`) und Umgebungsvariablen
- Getter-Methode für GitHub-Token hinzugefügt
2026-09-03 17:21:04 +02:00
DragonSlayer_14 14a1c4944a Test: GitHubClient-Token-Handling verbessert 2026-09-02 23:40:23 +02:00
DragonSlayer_14 c743f51526 Feat: Fügt Utils-Modul hinzu und integriert Umgebungsvariablen in Config 2026-09-02 22:31:31 +02:00
DragonSlayer_14 f4160df1d7 Merge pull request 'Merge testing in main: Release 1.0.1' (#9) from testing into main
Main Release & Publish / Build, Publish Packages (Stable) & Create Release (push) Successful in 9m24s
Reviewed-on: #9
2026-08-28 15:53:23 +00:00
DragonSlayer_14 3f5f18d98f Merge pull request 'Merge dev in testing: Ändert Container-Image auf Debian Trixie' (#8) from dev into testing
Testing Build, Publish & Preview Release / Build, Publish Packages (Testing) & Create Preview Release (push) Successful in 10m3s
Reviewed-on: #8
2026-08-28 15:34:32 +00:00
DragonSlayer_14 85e9046768 Merge remote-tracking branch 'origin/main' into dev 2026-08-28 17:34:16 +02:00
DragonSlayer_14 5728649c79 Feat: Fügt qodana.yaml hinzu 2026-08-28 17:33:34 +02:00
DragonSlayer_14 8eb0048606 Feat: Hebt Version auf 1.0.1 2026-08-28 17:27:40 +02:00
DragonSlayer_14 d92cb57d87 Feat: Ändert Debian von Bookworm auf Trixie 2026-08-28 17:27:06 +02:00
DragonSlayer_14 884f2dfec1 Merge pull request 'Merge testing in main: Release 1.0.0' (#7) from testing into main
Main Release & Publish / Build, Publish Packages (Stable) & Create Release (push) Successful in 9m5s
Reviewed-on: #7
2026-08-28 15:15:12 +00:00
DragonSlayer_14 e7407da5b4 Merge pull request 'Merge dev in testing: Feat: Hebt Version auf 1.0.0 an' (#6) from dev into testing
Testing Build, Publish & Preview Release / Build, Publish Packages (Testing) & Create Preview Release (push) Successful in 9m35s
Reviewed-on: #6
2026-08-28 15:03:54 +00:00
DragonSlayer_14 0f28e1bdbc Feat: Hebt Version auf 1.0.0 an 2026-08-28 17:03:19 +02:00
DragonSlayer_14 bab42a5e88 Merge pull request 'Merge dev in testing: Feat: Passt assets an und aktualisiert Paket-Abhängigkeiten.' (#5) from dev into testing
Testing Build, Publish & Preview Release / Build, Publish Packages (Testing) & Create Preview Release (push) Successful in 9m35s
Reviewed-on: #5
2026-08-28 12:39:20 +00:00
DragonSlayer_14 65785eac76 Feat: Hebt Version auf 0.1.2 2026-08-28 14:38:42 +02:00
DragonSlayer_14 47c6567ec7 Feat: Passt assets an und aktualisiert Paket-Abhängigkeiten. 2026-08-28 14:38:21 +02:00
DragonSlayer_14 52964e28f0 Feat: Ändert Dokumentationssprache auf Deutsch 2026-08-27 23:57:56 +02:00
DragonSlayer_14 9441d647a8 Fix: Formatiert Tabelle korrekt 2026-08-27 23:46:58 +02:00
DragonSlayer_14 966bcc3778 Merge pull request 'Merge dev in testing: Passt .idea-Files an' (#4) from dev into testing
Testing Build, Publish & Preview Release / Build, Publish Packages (Testing) & Create Preview Release (push) Successful in 8m56s
Reviewed-on: #4
2026-08-27 21:43:45 +00:00
DragonSlayer_14 01e511cdbb Merge pull request 'Merge dev in testing: Feat: Hebt Version auf 0.1.1 an' (#3) from dev into testing
Testing Build, Publish & Preview Release / Build, Publish Packages (Testing) & Create Preview Release (push) Failing after 9m3s
Reviewed-on: #3
2026-08-27 20:38:51 +00:00
17 changed files with 565 additions and 269 deletions
Generated
+1 -1
View File
@@ -866,7 +866,7 @@ dependencies = [
[[package]]
name = "mirror-package"
version = "0.1.1"
version = "1.0.2"
dependencies = [
"anyhow",
"clap",
+16 -8
View File
@@ -1,12 +1,12 @@
[package]
name = "mirror-package"
version = "0.1.1"
version = "1.0.2"
edition = "2024"
authors = ['DragonSlayer_14']
readme = "README.md"
license = "GPL-3.0-or-later"
repository = "https://gitea.creative-dragonslayer.de/Linuxapps/MirrorPackage"
description = "Mirror prebuilt Linux packages from GitHub Releases to Gitea Package Registry"
description = "Spiegelt vorkompilierte Linux-Pakete aus GitHub-Releases in Gitea-Paket-Registries"
[dependencies]
config-ctdra = { version = "1.0.4", registry = "gitea" }
@@ -29,18 +29,26 @@ maintainer = "DragonSlayer_14"
copyright = "2026 DragonSlayer_14"
section = "utils"
priority = "optional"
depends = "$auto"
depends = "$auto, ca-certificates"
extended-description = """\
Mirror prebuilt Linux packages (.deb, .rpm, .pkg.tar.zst) from GitHub Releases into Gitea Package Registries.\
Spiegelt vorkompilierte Linux-Pakete (.deb, .rpm, .pkg.tar.zst) aus GitHub-Releases in Gitea-Paket-Registries.\
"""
assets = []
assets = [
["target/release/mirror-package", "usr/bin/mirror-package", "755"],
["README.md", "usr/share/doc/mirror-package/README.md", "644"],
["LICENSE", "usr/share/doc/mirror-package/copyright", "644"],
]
[package.metadata.generate-rpm]
assets = []
requires = { }
assets = [
{ source = "target/release/mirror-package", dest = "/usr/bin/mirror-package", mode = "755" },
{ source = "README.md", dest = "/usr/share/doc/mirror-package/README.md", mode = "644", doc = true },
{ source = "LICENSE", dest = "/usr/share/licenses/mirror-package/LICENSE", mode = "644", license = true },
]
requires = { "ca-certificates" = "*", "glibc" = "*" }
[package.metadata.arch]
pkgrel = "1"
arch = "x86_64"
depends = ["gcc-libs", "glibc"]
depends = ["gcc-libs", "glibc", "ca-certificates"]
optdepends = []
+7 -7
View File
@@ -1,9 +1,9 @@
# syntax=docker/dockerfile:1
# Minimal and secure runtime image
FROM debian:bookworm-slim AS runtime
# Minimales und gehärtetes Runtime-Image
FROM debian:trixie-slim AS runtime
# Install CA certificates and minimal runtime dynamic libraries
# CA-Zertifikate und minimale dynamische Laufzeitbibliotheken installieren
RUN apt-get update && \
apt-get install -y --no-install-recommends \
ca-certificates \
@@ -19,18 +19,18 @@ RUN apt-get update && \
/etc/mirror-package && \
chown -R appuser:appuser /home/appuser /etc/mirror-package
# Build argument pointing to the prebuilt binary
# Build-Argument mit Verweis auf die vorkompilierte Binary
ARG TARGET_BIN=target/release/mirror-package
COPY ${TARGET_BIN} /usr/local/bin/mirror-package
# Set strict executable permissions
# Strikte Ausführungsrechte setzen
RUN chmod 0755 /usr/local/bin/mirror-package
# Run as unprivileged user
# Als unprivilegierter Benutzer ausführen
USER 10001:10001
WORKDIR /home/appuser
# Expose volume mount points for configuration and persistent state/logs
# Volume-Mount-Punkte für Konfiguration und persistenten Status / Logs exponieren
VOLUME ["/home/appuser/.config/mirror-package", "/home/appuser/.local/state/mirror-package"]
ENTRYPOINT ["/usr/local/bin/mirror-package"]
+96 -97
View File
@@ -1,131 +1,131 @@
# mirror-package
Automated tool to extract, download, and mirror prebuilt Linux packages from GitHub Releases into a self-hosted Gitea / Forgejo Package Registry.
Automatisiertes Werkzeug zum Extrahieren, Herunterladen und Spiegeln vorkompilierter Linux-Pakete aus GitHub-Releases in eine selbstgehostete Gitea- / Forgejo-Paket-Registry.
`mirror-package` tracks configured GitHub repositories (such as `raspberrypi/rpi-imager` or `Heroic-Games-Launcher/HeroicGamesLauncher`), identifies precompiled Linux packages (`.deb`, `.rpm`, `.pkg.tar.zst`, `.pkg.tar.xz`, `.pkg.tar.gz`, `.pacman`), and automatically publishes them into the appropriate Gitea Package Registry distributions according to release stability rules.
`mirror-package` überwacht konfigurierte GitHub-Repositories (wie z. B. `raspberrypi/rpi-imager` oder `Heroic-Games-Launcher/HeroicGamesLauncher`), identifiziert vorkompilierte Linux-Pakete (`.deb`, `.rpm`, `.pkg.tar.zst`, `.pkg.tar.xz`, `.pkg.tar.gz`, `.pacman`) und veröffentlicht diese anhand von Release-Stabilitätsregeln automatisch in den passenden Distributionen der Gitea-Paket-Registry.
---
## Features
## Funktionen
- **Multi-Distribution Package Support**:
- **Unterstützung mehrerer Distributionen**:
- **Debian / Ubuntu** (`.deb`)
- **Fedora / RHEL / openSUSE** (`.rpm`)
- **Arch Linux** (`.pkg.tar.zst`, `.pkg.tar.xz`, `.pkg.tar.gz`, `.pacman`)
- **Distribution Routing Rules**:
- **Stable Releases**: Debian packages are published to **both** `stable` and `testing` Debian pools (`pool/stable/main` and `pool/testing/main`); RPM packages to `rpm/stable`; Arch packages to `arch/stable`.
- **Pre-Releases**: Published **only** to testing channels: Debian `pool/testing/main`, RPM `rpm/testing`, and Arch `arch/testing`.
- **Persistent Configuration with `config-ctdra`**:
- Automatically stores settings in user (`~/.config/mirror-package/config.toml`) or system (`/etc/mirror-package/config.toml`) paths.
- Adding or removing repositories via CLI automatically updates the configuration file.
- **Flexible Authentication**:
- GitHub releases can be queried anonymously (no token required).
- Optional GitHub Personal Access Token support for higher API rate limits.
- Gitea instance URL, API token, and registry owner configurable via CLI flags, environment variables, or config file.
- **Diagnostics & Dry-Run Mode**:
- Unified file and console logging powered by `logger-ctdra`.
- `--dry-run` flag to inspect download and upload steps without making remote changes.
- **Distributions-Routing-Regeln**:
- **Stabile Releases**: Debian-Pakete werden **sowohl** im `stable`- als auch im `testing`-Debian-Pool veröffentlicht (`pool/stable/main` und `pool/testing/main`); RPM-Pakete unter `rpm/stable`; Arch-Pakete unter `arch/stable`.
- **Pre-Releases**: Werden **ausschließlich** in Testing-Kanälen veröffentlicht: Debian `pool/testing/main`, RPM `rpm/testing` und Arch `arch/testing`.
- **Persistente Konfiguration über `config-ctdra`**:
- Speichert Einstellungen automatisch im Benutzerpfad (`~/.config/mirror-package/config.toml`) oder Systempfad (`/etc/mirror-package/config.toml`).
- Das Hinzufügen oder Entfernen von Repositories über die CLI aktualisiert automatisch die Konfigurationsdatei.
- **Flexible Authentifizierung**:
- GitHub-Releases können anonym abgefragt werden (kein Token erforderlich).
- Optionale Unterstützung für GitHub Personal Access Tokens zur Vermeidung von API-Rate-Limits.
- Gitea-Instanz-URL, API-Token und Registry-Owner können per CLI-Flags, Umgebungsvariablen oder Konfigurationsdatei festgelegt werden.
- **Diagnose & Dry-Run-Modus**:
- Einheitliches Datei- und Konsolen-Logging über `logger-ctdra`.
- `--dry-run`-Flag zur Simulation von Download- und Upload-Schritten ohne Änderungen an Remote-Systemen vorzunehmen.
---
## CLI Usage & Commands
## CLI-Verwendung & Befehle
### 1. Configuration (`config`)
Set up your Gitea credentials and optional GitHub token:
### 1. Konfiguration (`config`)
Richte deine Gitea-Zugangsdaten und das optionale GitHub-Token ein:
```bash
# Configure Gitea instance and registry owner
# Gitea-Instanz und Registry-Owner konfigurieren
mirror-package config set \
--gitea-url "https://gitea.creative-dragonslayer.de" \
--gitea-token "your_gitea_api_token" \
--gitea-token "dein_gitea_api_token" \
--registry-owner "Linuxapps"
# Optional: Configure GitHub Token for rate limits
# Optional: GitHub-Token für höhere API-Rate-Limits konfigurieren
mirror-package config set --github-token "ghp_xxxxxxxxxxxx"
# Display current configuration
# Aktuelle Konfiguration anzeigen
mirror-package config show
```
Credentials can also be passed via environment variables (`GITEA_URL`, `GITEA_TOKEN`, `REGISTRY_OWNER`, `GITHUB_TOKEN`) or global CLI arguments.
Zugangsdaten können auch über Umgebungsvariablen (`GITEA_URL`, `GITEA_TOKEN`, `REGISTRY_OWNER`, `GITHUB_TOKEN`) oder globale CLI-Argumente übergeben werden.
### 2. Managing Monitored Repositories (`add`, `remove`, `list`)
### 2. Überwachte Repositories verwalten (`add`, `remove`, `list`)
```bash
# Add repositories to persistent configuration
# Repositories zur persistenten Konfiguration hinzufügen
mirror-package add raspberrypi/rpi-imager
mirror-package add Heroic-Games-Launcher/HeroicGamesLauncher
# Add repository without pre-release syncing
# Repository ohne Synchronisierung von Pre-Releases hinzufügen
mirror-package add some-owner/some-repo --no-prereleases
# List configured repositories and their sync status
# Konfigurierte Repositories und deren Sync-Status auflisten
mirror-package list
# Remove a repository
# Ein Repository entfernen
mirror-package remove raspberrypi/rpi-imager
```
### 3. Synchronizing Packages (`sync`)
### 3. Pakete synchronisieren (`sync`)
```bash
# Sync all configured repositories (latest stable and latest pre-release)
# Alle konfigurierten Repositories synchronisieren (neueste stabile Version und neuestes Pre-Release)
mirror-package sync
# Sync a specific repository
# Ein bestimmtes Repository synchronisieren
mirror-package sync raspberrypi/rpi-imager
# Sync all historical releases for a repository
# Alle historischen Releases eines Repositories synchronisieren
mirror-package sync raspberrypi/rpi-imager --history
# Dry-run test (simulates without uploading)
# Dry-Run-Test (Simulation ohne Upload)
mirror-package sync --dry-run
```
---
## Docker & Container Usage
## Docker & Container-Nutzung
`mirror-package` is packaged and published as an ultra-secure, lightweight container image to the Gitea Container Registry.
`mirror-package` wird als minimales, gehärtetes Container-Image in der Gitea Container Registry bereitgestellt.
### 1. Image Registry & Tagging Strategy
### 1. Image-Registry & Tagging-Strategie
- **Stable Releases (`main` branch)**:
- **Stabile Releases (`main`-Branch)**:
- `<registry>/<owner>/mirror-package:latest`
- `<registry>/<owner>/mirror-package:<version>`
- `<registry>/<owner>/mirror-package:v<version>`
- `<registry>/<owner>/mirror-package:<version>.<build_number>`
- **Testing & Preview Releases (`testing` branch)**:
- **Testing- & Vorschau-Releases (`testing`-Branch)**:
- `<registry>/<owner>/mirror-package:testing`
- `<registry>/<owner>/mirror-package:<version>-preview`
- `<registry>/<owner>/mirror-package:<version>-testing`
- `<registry>/<owner>/mirror-package:<version>-preview.<build_number>`
- `<registry>/<owner>/mirror-package:testing-<build_number>`
- *(Note: The `:latest` tag is strictly reserved for the stable `main` workflow).*
- *(Hinweis: Der Tag `:latest` ist strikt dem stabilen `main`-Workflow vorbehalten).*
### 2. Persistent Storage Paths
### 2. Persistente Speicherpfade
To persist configurations, tracked repository states, and logs across container restarts, mount the following container paths to host directories or Docker volumes:
Um Konfigurationen, Status und Protokolle über Container-Neustarts hinweg beizubehalten, binde folgende Pfade an Host-Verzeichnisse oder Docker-Volumes:
| Container Path | Purpose | Recommended Mount Type |
|---|---|---|
| `/home/appuser/.config/mirror-package` | Stores `config.toml` (credentials & repository list) | Host Directory / Volume (rw) |
| `/home/appuser/.local/state/mirror-package` | Persistent application logs & state files | Host Directory / Volume (rw) |
| `/tmp` | Download & stream buffer for package binaries | `tmpfs` (RAM / tempfs, rw, noexec) |
| Container-Pfad | Zweck | Empfohlener Mount-Typ |
|---------------------------------------------|-------------------------------------------------------|------------------------------------|
| `/home/appuser/.config/mirror-package` | Speichert `config.toml` (Zugangsdaten & Repo-Liste) | Host-Verzeichnis / Volume (rw) |
| `/home/appuser/.local/state/mirror-package` | Persistente Anwendungsprotokolle & Statusdateien | Host-Verzeichnis / Volume (rw) |
| `/tmp` | Download- & Streaming-Puffer für Paketdateien | `tmpfs` (RAM / tempfs, rw, noexec) |
### 3. Security Hardening
### 3. Sicherheitshärtung
The container is built with security as the highest priority:
- **Unprivileged User**: Runs as `appuser` (UID `10001`, GID `10001`), never as `root`.
- **Read-Only Root Filesystem**: Fully operational with `--read-only` / `read_only: true`.
- **No Capabilities**: All Linux capabilities can be safely dropped (`--cap-drop=ALL`).
- **No Privilege Escalation**: Enforces `no-new-privileges:true`.
- **Minimal Image Size**: Based on Debian Bookworm slim, containing only CA certificates and necessary shared libraries (~40 MB).
Der Container wurde nach höchsten Sicherheitsstandards aufgebaut:
- **Unprivilegierter Benutzer**: Läuft als `appuser` (UID `10001`, GID `10001`), niemals als `root`.
- **Read-Only Root-Dateisystem**: Voll funktionsfähig mit `--read-only` / `read_only: true`.
- **Keine Capabilities**: Sämtliche Linux-Capabilities können sicher entzogen werden (`--cap-drop=ALL`).
- **Keine Rechteausweitung**: Erzwingt `no-new-privileges:true`.
- **Minimale Image-Größe**: Basiert auf Debian Trixie Slim und enthält nur CA-Zertifikate und notwendige dynamische Bibliotheken (~40 MB).
### 4. Running via Docker CLI
### 4. Ausführung über Docker CLI
```bash
# Initialize / configure credentials
# Zugangsdaten initialisieren / konfigurieren
docker run --rm \
--name mirror-package \
--read-only \
@@ -137,10 +137,10 @@ docker run --rm \
gitea.creative-dragonslayer.de/linuxapps/mirror-package:latest \
config set \
--gitea-url "https://gitea.creative-dragonslayer.de" \
--gitea-token "your_token" \
--gitea-token "dein_token" \
--registry-owner "Linuxapps"
# Add repositories
# Repositories hinzufügen
docker run --rm \
--read-only \
--cap-drop=ALL \
@@ -151,7 +151,7 @@ docker run --rm \
gitea.creative-dragonslayer.de/linuxapps/mirror-package:latest \
add raspberrypi/rpi-imager
# Run synchronization (can be triggered by a host cronjob or scheduler)
# Synchronisation ausführen (kann über Cronjob oder Scheduler auf dem Host getriggert werden)
docker run --rm \
--read-only \
--cap-drop=ALL \
@@ -163,7 +163,7 @@ docker run --rm \
sync
```
### 5. Docker Compose Example (`docker-compose.yml`)
### 5. Docker Compose Beispiel (`docker-compose.yml`)
```yaml
services:
@@ -183,72 +183,71 @@ services:
- /tmp:rw,noexec,nosuid,size=2G
environment:
- GITEA_URL=https://gitea.creative-dragonslayer.de
- GITEA_TOKEN=your_gitea_api_token
- GITEA_TOKEN=dein_gitea_api_token
- REGISTRY_OWNER=Linuxapps
- GITHUB_TOKEN=your_optional_github_pat
- GITHUB_TOKEN=dein_optionaler_github_pat
- LOG_LEVEL=info
command: ["sync"]
```
---
## Repository Structure
## Repository-Struktur
```text
├── .cargo/
│ └── config.toml # Linker & Cargo configuration
│ └── config.toml # Linker- & Cargo-Konfiguration
├── .gitea/
│ └── workflows/
│ ├── main.yaml # CI/CD: Release, Packages & Container (Stable)
│ └── testing.yaml # CI/CD: Preview, Packages & Container (Testing)
│ ├── main.yaml # CI/CD: Release, Pakete & Container (Stable)
│ └── testing.yaml # CI/CD: Preview, Pakete & Container (Testing)
├── scripts/
│ ├── get-build-number.py # Dynamic build/revision number resolution
│ └── package-arch.py # Native Arch Linux package builder
│ ├── get-build-number.py # Dynamische Ermittlung der Build-/Revisionsnummer
│ └── package-arch.py # Erstellung nativer Arch Linux-Pakete
├── src/
│ ├── main.rs # CLI entrypoint & subcommand dispatch
│ ├── lib.rs # Library root & module exports
│ ├── cli.rs # Clap CLI arguments & options
│ ├── config.rs # config-ctdra integration & models
│ ├── gitea.rs # Gitea Package Registry upload client
│ ├── github.rs # GitHub API client & package classifier
│ └── pipeline.rs # End-to-end sync workflow & temp storage
│ ├── main.rs # CLI-Einstiegspunkt & Befehlsausführung
│ ├── lib.rs # Bibliotheks-Wurzel & Modulexporte
│ ├── cli.rs # Clap-CLI-Argumente & Optionen
│ ├── config.rs # config-ctdra Anbindung & Konfigurationsmodelle
│ ├── gitea.rs # Gitea-Paket-Registry Upload-Client
│ ├── github.rs # GitHub-API-Client & Paket-Klassifizierung
│ └── pipeline.rs # End-to-End-Synchronisationspipeline & temporärer Speicher
├── tests/
│ ├── config_tests.rs # Tests for configuration management
│ ├── gitea_tests.rs # Tests for Gitea upload routing
│ └── github_tests.rs # Tests for package classification & parsing
├── .dockerignore # Container build ignore rules
├── Dockerfile # Secure minimal runtime container
├── docker-compose.example.yml # Example Docker Compose configuration
├── Cargo.toml # Project manifest and packaging metadata
├── LICENSE # GPL-3.0-or-later License
├── AGENTS.md # Agent & developer guidelines
└── README.md # Documentation
│ ├── config_tests.rs # Tests für die Konfigurationsverwaltung
│ ├── gitea_tests.rs # Tests für das Gitea-Upload-Routing
│ └── github_tests.rs # Tests für Paket-Klassifizierung & Parsing
├── .dockerignore # Ausschlussregeln für Container-Builds
├── Dockerfile # Gehärtetes, minimales Runtime-Container-Image
├── docker-compose.example.yml # Beispielkonfiguration für Docker Compose
├── Cargo.toml # Projekt-Manifest und Paketierungs-Metadaten
├── LICENSE # GPL-3.0-or-later Lizenztext
├── AGENTS.md # Agenten- & Entwickler-Richtlinien
└── README.md # Projektdokumentation
```
---
## Local Development & Building
## Lokale Entwicklung & Bauen
### Prerequisites
- **Rust & Cargo** (Stable toolchain, Edition 2024 supported)
- **Python 3** (for packaging scripts)
### Voraussetzungen
- **Rust & Cargo** (Stable Toolchain, Edition 2024 unterstützt)
- **Python 3** (für Paketierungsskripte)
### Build & Test Commands
### Befehle zum Bauen & Testen
```bash
# Run syntax and type checks
# Syntax- und Typprüfung ausführen
cargo check
# Run unit tests
# Unit-Tests ausführen
cargo test
# Build release binary
# Release-Binary kompilieren
cargo build --release
```
---
## License
This project is licensed under the [GPL-3.0-or-later](LICENSE) license.
## Lizenz
Dieses Projekt ist unter der [GPL-3.0-or-later](LICENSE)-Lizenz lizenziert.
+4 -4
View File
@@ -2,14 +2,14 @@ services:
mirror-package:
image: gitea.creative-dragonslayer.de/linuxapps/mirror-package:latest
container_name: mirror-package
# Security: run as unprivileged user, read-only rootfs, drop capabilities
# Sicherheit: Als unprivilegierter Benutzer ausführen, Read-Only-Dateisystem, Capabilities entziehen
user: "10001:10001"
read_only: true
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
# Persistent storage and temporary stream buffer
# Persistenter Speicher und temporärer Streaming-Puffer
volumes:
- ./config:/home/appuser/.config/mirror-package
- ./state:/home/appuser/.local/state/mirror-package
@@ -17,8 +17,8 @@ services:
- /tmp:rw,noexec,nosuid,size=2G
environment:
- GITEA_URL=https://gitea.creative-dragonslayer.de
- GITEA_TOKEN=your_gitea_api_token
- GITEA_TOKEN=dein_gitea_api_token
- REGISTRY_OWNER=Linuxapps
- GITHUB_TOKEN=your_optional_github_pat
- GITHUB_TOKEN=dein_optionaler_github_pat
- LOG_LEVEL=info
command: ["sync"]
+50
View File
@@ -0,0 +1,50 @@
#-------------------------------------------------------------------------------#
# Qodana analysis is configured by qodana.yaml file #
# https://www.jetbrains.com/help/qodana/qodana-yaml.html #
#-------------------------------------------------------------------------------#
#################################################################################
# WARNING: Do not store sensitive information in this file, #
# as its contents will be included in the Qodana report. #
#################################################################################
version: "1.0"
#Specify inspection profile for code analysis
profile:
name: qodana.starter
#Enable inspections
#include:
# - name: <SomeEnabledInspectionId>
#Disable inspections
#exclude:
# - name: <SomeDisabledInspectionId>
# paths:
# - <path/where/not/run/inspection>
#Execute shell command before Qodana execution (Applied in CI/CD pipeline)
#bootstrap: sh ./prepare-qodana.sh
#Install IDE plugins before Qodana execution (Applied in CI/CD pipeline)
#plugins:
# - id: <plugin.id> #(plugin id can be found at https://plugins.jetbrains.com)
# Quality gate. Will fail the CI/CD pipeline if any condition is not met
# severityThresholds - configures maximum thresholds for different problem severities
# testCoverageThresholds - configures minimum code coverage on a whole project and newly added code
# dependencyLicenses - fails the run on prohibited or unknown dependency licenses
# Code Coverage is available in Ultimate and Ultimate Plus plans
#failureConditions:
# severityThresholds:
# any: 15
# critical: 5
# testCoverageThresholds:
# fresh: 70
# total: 50
# dependencyLicenses:
# failOnProhibited: true
# failOnUnknown: false
#Specify Qodana linter for analysis (Applied in CI/CD pipeline)
linter: jetbrains/qodana-<linter>:2026.2
+27 -27
View File
@@ -1,41 +1,41 @@
use clap::{Args, Parser, Subcommand, ValueEnum};
use std::path::PathBuf;
/// Command line parser for mirror-package.
/// Befehlszeilen-Parser für mirror-package.
#[derive(Parser, Debug)]
#[command(
name = "mirror-package",
author,
version,
about = "Mirror Linux packages (.deb, .rpm, .pkg.tar.zst) from GitHub Releases to Gitea Package Registry",
about = "Spiegelt Linux-Pakete (.deb, .rpm, .pkg.tar.zst) aus GitHub-Releases in Gitea-Paket-Registries",
long_about = None
)]
pub struct Cli {
/// Gitea instance base URL (e.g. https://gitea.creative-dragonslayer.de)
/// Basis-URL der Gitea-Instanz (z. B. https://gitea.creative-dragonslayer.de)
#[arg(long, env = "GITEA_URL", global = true)]
pub gitea_url: Option<String>,
/// Gitea API Token with package write permissions
/// Gitea-API-Token mit Schreibberechtigung für Pakete
#[arg(long, env = "GITEA_TOKEN", global = true)]
pub gitea_token: Option<String>,
/// Gitea Package Registry owner / organization
/// Gitea-Paket-Registry-Owner / Organisation
#[arg(long, env = "REGISTRY_OWNER", global = true)]
pub registry_owner: Option<String>,
/// Optional GitHub Token for API requests (avoids rate limits)
/// Optionales GitHub-Token für API-Anfragen (vermeidet Rate-Limits)
#[arg(long, env = "GITHUB_TOKEN", global = true)]
pub github_token: Option<String>,
/// Custom path to configuration file
/// Benutzerdefinierter Pfad zur Konfigurationsdatei
#[arg(long, global = true)]
pub config: Option<PathBuf>,
/// Perform a dry run without downloading or uploading packages
/// Führt einen Probelauf ohne Herunterladen oder Hochladen von Paketen durch
#[arg(long, global = true)]
pub dry_run: bool,
/// Logging level (error, warn, info, debug)
/// Logging-Level (error, warn, info, debug)
#[arg(long, value_enum, default_value_t = LogLevelArg::Info, global = true)]
pub log_level: LogLevelArg,
@@ -64,59 +64,59 @@ impl From<LogLevelArg> for logger_ctdra::LogLevel {
#[derive(Subcommand, Debug)]
pub enum Commands {
/// Synchronize packages from GitHub Releases to Gitea Package Registry
/// Synchronisiert Pakete aus GitHub-Releases in die Gitea-Paket-Registry
Sync(SyncArgs),
/// Add a GitHub repository to the persistent configuration
/// Fügt ein GitHub-Repository zur persistenten Konfiguration hinzu
Add(AddArgs),
/// Remove a GitHub repository from the persistent configuration
/// Entfernt ein GitHub-Repository aus der persistenten Konfiguration
#[command(alias = "rm")]
Remove(RemoveArgs),
/// List configured repositories and their sync status
/// Listet konfigurierte Repositories und deren Sync-Status auf
#[command(alias = "ls")]
List,
/// View or update application configuration
/// Zeigt die Anwendungskonfiguration an oder aktualisiert sie
Config(ConfigArgs),
}
#[derive(Args, Debug)]
pub struct SyncArgs {
/// Specific repository name or URL to sync (e.g. "raspberrypi/rpi-imager"). If omitted, all configured repositories are synced.
/// Bestimmter Repository-Name oder URL zur Synchronisation (z. B. "raspberrypi/rpi-imager"). Wenn weggelassen, werden alle konfigurierten Repositories synchronisiert.
pub repo: Option<String>,
/// Sync all configured repositories
/// Alle konfigurierten Repositories synchronisieren
#[arg(short = 'a', long)]
pub all: bool,
/// Scan historical releases instead of only the latest release(s)
/// Historische Releases scannen anstatt nur die neuesten Release(s)
#[arg(long)]
pub history: bool,
/// Include pre-releases during this sync run (defaults to repository configuration if not specified)
/// Pre-Releases während dieses Synchronisationslaufs einbeziehen (Standard: Repository-Konfiguration)
#[arg(long)]
pub prereleases: Option<bool>,
}
#[derive(Args, Debug)]
pub struct AddArgs {
/// GitHub repository to mirror (e.g. "raspberrypi/rpi-imager" or full GitHub URL)
/// Zu spiegelndes GitHub-Repository (z. B. "raspberrypi/rpi-imager" oder vollständige GitHub-URL)
pub repo: String,
/// Do not mirror pre-releases for this repository
/// Keine Pre-Releases für dieses Repository spiegeln
#[arg(long)]
pub no_prereleases: bool,
/// Immediately synchronize this repository after adding
/// Dieses Repository nach dem Hinzufügen sofort synchronisieren
#[arg(long)]
pub sync: bool,
}
#[derive(Args, Debug)]
pub struct RemoveArgs {
/// Repository to remove (e.g. "raspberrypi/rpi-imager")
/// Zu entfernendes Repository (z. B. "raspberrypi/rpi-imager")
pub repo: String,
}
@@ -128,20 +128,20 @@ pub struct ConfigArgs {
#[derive(Subcommand, Debug)]
pub enum ConfigAction {
/// Show current configuration and configuration file location
/// Aktuelle Konfiguration und Speicherort der Konfigurationsdatei anzeigen
Show,
/// Set configuration parameters
/// Konfigurationsparameter setzen
Set {
/// Gitea instance base URL
/// Basis-URL der Gitea-Instanz
#[arg(long)]
gitea_url: Option<String>,
/// Gitea API Token
/// Gitea-API-Token
#[arg(long)]
gitea_token: Option<String>,
/// Gitea Package Registry owner
/// Gitea-Paket-Registry-Owner
#[arg(long)]
registry_owner: Option<String>,
+63 -35
View File
@@ -1,3 +1,4 @@
use crate::utils::{clean_repo_input, sanitize_string};
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
@@ -5,15 +6,15 @@ fn default_true() -> bool {
true
}
/// Configuration for an individual repository to mirror.
/// Konfiguration für ein einzelnes zu spiegelndes Repository.
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub struct RepoConfig {
/// Full name of the repository in "owner/repo" format.
/// Vollständiger Name des Repositories im Format "owner/repo".
pub name: String,
/// Whether to mirror pre-releases.
/// Gibt an, ob Pre-Releases gespiegelt werden sollen.
#[serde(default = "default_true")]
pub include_prereleases: bool,
/// Last successfully synced tag name.
/// Zuletzt erfolgreich synchronisierter Tag-Name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_synced_tag: Option<String>,
}
@@ -21,79 +22,75 @@ pub struct RepoConfig {
impl RepoConfig {
pub fn new(name: impl Into<String>, include_prereleases: bool) -> Self {
Self {
name: name.into(),
name: AppConfig::normalize_repo_name(&name.into()),
include_prereleases,
last_synced_tag: None,
}
}
}
/// Persistent application configuration.
/// Persistente Anwendungskonfiguration.
#[derive(Serialize, Deserialize, Clone, Debug, Default, PartialEq)]
pub struct AppConfig {
/// Gitea instance base URL (e.g. https://gitea.example.com).
/// Basis-URL der Gitea-Instanz (z. B. https://gitea.example.com).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gitea_url: Option<String>,
/// Gitea API authentication token.
/// Gitea-API-Authentifizierungstoken.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gitea_token: Option<String>,
/// Gitea Package Registry owner/organization name.
/// Name des Gitea-Paket-Registry-Owners bzw. der Organisation.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub registry_owner: Option<String>,
/// Optional GitHub Personal Access Token for increased rate limits.
/// Optionales GitHub Personal Access Token für höhere Rate-Limits.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub github_token: Option<String>,
/// Configured repositories to mirror.
/// Konfigurierte Repositories zur Spiegelung.
#[serde(default)]
pub repositories: Vec<RepoConfig>,
}
impl AppConfig {
/// Normalizes repository input, stripping leading/trailing slashes and GitHub URL prefixes.
/// Normalisiert die Repository-Eingabe, indem führende/nachgestellte Schrägstriche,
/// Query-Parameter, Fragmente und GitHub-URL-Präfixe entfernt werden.
pub fn normalize_repo_name(input: &str) -> String {
let trimmed = input.trim();
let cleaned = trimmed
.trim_start_matches("https://github.com/")
.trim_start_matches("http://github.com/")
.trim_start_matches("github.com/")
.trim_end_matches(".git")
.trim_matches('/');
cleaned.to_string()
clean_repo_input(input).to_string()
}
/// Adds or updates a repository in the configuration.
pub fn add_or_update_repo(&mut self, repo: RepoConfig) {
if let Some(existing) = self.repositories.iter_mut().find(|r| r.name.eq_ignore_ascii_case(&repo.name)) {
/// Fügt ein Repository zur Konfiguration hinzu oder aktualisiert ein bestehendes.
pub fn add_or_update_repo(&mut self, mut repo: RepoConfig) {
repo.name = Self::normalize_repo_name(&repo.name);
if let Some(existing) = self.repositories.iter_mut().find(|r| Self::normalize_repo_name(&r.name).eq_ignore_ascii_case(&repo.name)) {
existing.name = repo.name;
existing.include_prereleases = repo.include_prereleases;
} else {
self.repositories.push(repo);
}
}
/// Removes a repository by name. Returns true if it was found and removed.
/// Entfernt ein Repository anhand des Namens. Gibt true zurück, wenn es gefunden und entfernt wurde.
pub fn remove_repo(&mut self, repo_name: &str) -> bool {
let normalized = Self::normalize_repo_name(repo_name);
let before_len = self.repositories.len();
self.repositories.retain(|r| !r.name.eq_ignore_ascii_case(&normalized));
self.repositories.retain(|r| !Self::normalize_repo_name(&r.name).eq_ignore_ascii_case(&normalized));
self.repositories.len() < before_len
}
/// Finds a repository by name.
/// Sucht ein Repository anhand des Namens.
pub fn find_repo(&self, repo_name: &str) -> Option<&RepoConfig> {
let normalized = Self::normalize_repo_name(repo_name);
self.repositories.iter().find(|r| r.name.eq_ignore_ascii_case(&normalized))
self.repositories.iter().find(|r| Self::normalize_repo_name(&r.name).eq_ignore_ascii_case(&normalized))
}
/// Updates the last synced tag for a given repository.
/// Aktualisiert den zuletzt synchronisierten Tag für ein bestimmtes Repository.
pub fn update_last_synced_tag(&mut self, repo_name: &str, tag: String) {
let normalized = Self::normalize_repo_name(repo_name);
if let Some(repo) = self.repositories.iter_mut().find(|r| r.name.eq_ignore_ascii_case(&normalized)) {
if let Some(repo) = self.repositories.iter_mut().find(|r| Self::normalize_repo_name(&r.name).eq_ignore_ascii_case(&normalized)) {
repo.last_synced_tag = Some(tag);
}
}
}
/// Sets a custom path for the configuration file if provided.
/// Setzt einen benutzerdefinierten Pfad für die Konfigurationsdatei, falls angegeben.
pub fn init_config_path(custom_path: Option<&std::path::Path>) {
config_ctdra::set_config_name("config");
if let Some(path) = custom_path {
@@ -101,22 +98,53 @@ pub fn init_config_path(custom_path: Option<&std::path::Path>) {
}
}
/// Returns the resolved path of the configuration file.
/// Gibt den aufgelösten Pfad der Konfigurationsdatei zurück.
pub fn get_config_file_path() -> PathBuf {
config_ctdra::get_config_path()
}
/// Loads the application configuration. Falls back to defaults if not found.
/// Lädt die Anwendungskonfiguration. Verwendet Standardwerte, falls keine Datei vorhanden ist.
pub fn load_config() -> AppConfig {
config_ctdra::load_config::<AppConfig>()
let mut config = config_ctdra::load_config::<AppConfig>();
for repo in &mut config.repositories {
repo.name = AppConfig::normalize_repo_name(&repo.name);
}
/// Saves the application configuration to disk.
if let Ok(val) = std::env::var("GITEA_URL") {
let val = sanitize_string(&val);
if !val.is_empty() {
config.gitea_url = Some(val);
}
}
if let Ok(val) = std::env::var("GITEA_TOKEN") {
let val = sanitize_string(&val);
if !val.is_empty() {
config.gitea_token = Some(val);
}
}
if let Ok(val) = std::env::var("REGISTRY_OWNER") {
let val = sanitize_string(&val);
if !val.is_empty() {
config.registry_owner = Some(val);
}
}
if let Ok(val) = std::env::var("GITHUB_TOKEN") {
let val = sanitize_string(&val);
if !val.is_empty() {
config.github_token = Some(val);
}
}
config
}
/// Speichert die Anwendungskonfiguration auf der Festplatte.
pub fn save_config(config: &AppConfig) -> Result<(), config_ctdra::ConfyError> {
config_ctdra::store(config)
}
/// Modifies the application configuration in-place atomically and saves it.
/// Modifiziert die Anwendungskonfiguration atomar vor Ort und speichert sie.
pub fn modify_config<F>(f: F) -> Result<AppConfig, config_ctdra::ConfyError>
where
F: FnOnce(&mut AppConfig),
+6 -6
View File
@@ -3,7 +3,7 @@ use anyhow::{bail, Context, Result};
use std::path::Path;
use tokio::fs::File;
/// Credentials and target registry information for Gitea.
/// Zugangsdaten und Ziel-Registry-Informationen für Gitea.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct GiteaConfig {
pub base_url: String,
@@ -25,7 +25,7 @@ impl GiteaConfig {
}
}
/// Result of a package upload attempt.
/// Ergebnis eines Paket-Upload-Versuchs.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum UploadStatus {
Uploaded,
@@ -33,7 +33,7 @@ pub enum UploadStatus {
SimulatedDryRun,
}
/// Computes the target Gitea package registry upload URLs based on package type and release channel.
/// Berechnet die Ziel-Upload-URLs der Gitea-Paket-Registry basierend auf Pakettyp und Release-Kanal.
pub fn get_target_upload_urls(
base_url: &str,
owner: &str,
@@ -69,7 +69,7 @@ pub fn get_target_upload_urls(
}
}
/// Client for uploading packages to a Gitea Package Registry.
/// Client für den Upload von Paketen in eine Gitea-Paket-Registry.
#[derive(Clone, Debug)]
pub struct GiteaClient {
client: reqwest::Client,
@@ -77,7 +77,7 @@ pub struct GiteaClient {
}
impl GiteaClient {
/// Creates a new Gitea client.
/// Erstellt einen neuen Gitea-Client.
pub fn new(config: GiteaConfig) -> Result<Self> {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
@@ -93,7 +93,7 @@ impl GiteaClient {
Ok(Self { client, config })
}
/// Uploads a package file from disk to the specified Gitea endpoint.
/// Lädt eine Paketdatei von der Festplatte zum angegebenen Gitea-Endpunkt hoch.
pub async fn upload_file(
&self,
file_path: &Path,
+25 -24
View File
@@ -1,7 +1,7 @@
use anyhow::{bail, Context, Result};
use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
/// Supported Linux package distribution types.
/// Unterstützte Linux-Paketverteilungstypen.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum PackageType {
Debian,
@@ -10,10 +10,10 @@ pub enum PackageType {
}
impl PackageType {
/// Classifies an asset by its filename into a supported PackageType.
/// Klassifiziert ein Asset anhand seines Dateinamens in einen unterstützten PackageType.
pub fn from_filename(filename: &str) -> Option<Self> {
let name = filename.to_ascii_lowercase();
// Check Arch packages first due to multi-part extensions
// Arch-Pakete aufgrund mehrteiliger Dateiendungen zuerst prüfen
if name.ends_with(".pkg.tar.zst")
|| name.ends_with(".pkg.tar.xz")
|| name.ends_with(".pkg.tar.gz")
@@ -29,7 +29,7 @@ impl PackageType {
}
}
/// Returns human-readable name of the package type.
/// Gibt den lesbaren Namen des Pakettyps zurück.
pub fn name(&self) -> &'static str {
match self {
PackageType::Debian => "Debian (.deb)",
@@ -39,7 +39,7 @@ impl PackageType {
}
}
/// A classified Linux package asset from a release.
/// Ein klassifiziertes Linux-Paket-Asset aus einem Release.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ReleaseAsset {
pub name: String,
@@ -48,7 +48,7 @@ pub struct ReleaseAsset {
pub package_type: PackageType,
}
/// A GitHub release with extracted Linux package assets.
/// Ein GitHub-Release mit extrahierten Linux-Paket-Assets.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct GitHubRelease {
pub id: u64,
@@ -77,7 +77,7 @@ struct GhApiAsset {
size: u64,
}
/// Client for interacting with the GitHub API.
/// Client für die Interaktion mit der GitHub-API.
#[derive(Clone, Debug)]
pub struct GitHubClient {
client: reqwest::Client,
@@ -85,8 +85,9 @@ pub struct GitHubClient {
}
impl GitHubClient {
/// Creates a new GitHub API client with optional authentication token.
/// Erstellt einen neuen GitHub-API-Client mit optionalem Authentifizierungstoken.
pub fn new(token: Option<String>) -> Result<Self> {
let token = token.filter(|t| !t.trim().is_empty());
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::USER_AGENT,
@@ -105,10 +106,16 @@ impl GitHubClient {
Ok(Self { client, token })
}
/// Fetches releases for a given repository (format "owner/repo").
/// Gibt das optionale Authentifizierungstoken zurück (falls gesetzt).
pub fn token(&self) -> Option<&str> {
self.token.as_deref()
}
/// Ruft Releases für ein angegebenes Repository ab (Format "owner/repo").
///
/// If `history` is false, retrieves only the latest release(s).
/// If `include_prereleases` is false, skips pre-releases.
/// Wenn `history` false ist, werden nur die neuesten Releases abgerufen.
/// Wenn `include_prereleases` false ist, werden Pre-Releases übersprungen.
pub async fn fetch_releases(
&self,
repo: &str,
@@ -186,9 +193,9 @@ impl GitHubClient {
});
}
// If not scanning full history, stop after the first page (or once we found stable/prerelease)
// Wenn kein vollständiger Verlauf gescannt werden soll, nach der ersten Seite stoppen
if !history {
// Keep only the most recent stable release and most recent pre-release (if allowed)
// Nur das neueste stabile Release und neueste Pre-Release behalten (falls zulässig)
let mut selected = Vec::new();
let latest_stable = releases.iter().find(|r| !r.prerelease);
if let Some(stable) = latest_stable {
@@ -198,7 +205,7 @@ impl GitHubClient {
if include_prereleases {
let latest_prerelease = releases.iter().find(|r| r.prerelease);
if let Some(prerelease) = latest_prerelease {
// Avoid duplicates if stable and pre-release are identical (rare)
// Duplikate vermeiden, falls Stable und Pre-Release identisch sind
if !selected.iter().any(|r| r.id == prerelease.id) {
selected.push(prerelease.clone());
}
@@ -213,7 +220,7 @@ impl GitHubClient {
Ok(releases)
}
/// Downloads an asset from the given download URL as a streaming response.
/// Lädt ein Asset von der angegebenen Download-URL als Streaming-Response herunter.
pub async fn download_asset_stream(&self, download_url: &str) -> Result<reqwest::Response> {
let mut req = self.client.get(download_url);
if let Some(token) = &self.token {
@@ -237,15 +244,9 @@ impl GitHubClient {
}
}
/// Helper to parse "owner/repo" from repository string.
/// Hilfsfunktion zum Parsen von "owner/repo" aus einer Repository-Zeichenkette.
pub fn parse_repo_owner_name(repo: &str) -> Result<(&str, &str)> {
let cleaned = repo
.trim()
.trim_start_matches("https://github.com/")
.trim_start_matches("http://github.com/")
.trim_start_matches("github.com/")
.trim_end_matches(".git")
.trim_matches('/');
let cleaned = crate::utils::clean_repo_input(repo);
let parts: Vec<&str> = cleaned.split('/').collect();
if parts.len() != 2 || parts[0].is_empty() || parts[1].is_empty() {
+4
View File
@@ -1,5 +1,9 @@
//! `mirror-package` ist eine Anwendung zum Spiegeln vorkompilierter Linux-Pakete
//! (.deb, .rpm, .pkg.tar.zst) aus GitHub-Releases in Gitea- / Forgejo-Paket-Registries.
pub mod cli;
pub mod config;
pub mod gitea;
pub mod github;
pub mod pipeline;
pub mod utils;
+4 -4
View File
@@ -11,16 +11,16 @@ use mirror_package::pipeline::{sync_single_repository, SyncOptions, SyncReport};
async fn main() -> Result<()> {
let cli = Cli::parse();
// Initialize global logger level
// Globalen Logging-Level initialisieren
set_log_level(cli.log_level.into());
// Initialize custom config path if specified
// Benutzerdefinierten Konfigurationspfad initialisieren, falls angegeben
init_config_path(cli.config.as_deref());
// Load persistent configuration
// Persistente Konfiguration laden
let mut app_config = load_config();
// CLI flags override configuration values
// CLI-Flags überschreiben Konfigurationswerte
if let Some(url) = cli.gitea_url {
app_config.gitea_url = Some(url);
}
+10 -10
View File
@@ -8,7 +8,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
use tokio::fs::{remove_file, File};
use tokio::io::AsyncWriteExt;
/// Summary statistics of a synchronization operation.
/// Zusammenfassende Statistiken eines Synchronisationsvorgangs.
#[derive(Default, Debug, Clone, PartialEq, Eq)]
pub struct SyncReport {
pub repositories_processed: usize,
@@ -18,7 +18,7 @@ pub struct SyncReport {
pub packages_simulated: usize,
}
/// Options controlling a repository sync run.
/// Optionen zur Steuerung eines Repository-Synchronisationslaufs.
#[derive(Debug, Clone)]
pub struct SyncOptions {
pub history: bool,
@@ -26,7 +26,7 @@ pub struct SyncOptions {
pub dry_run: bool,
}
/// Executes the synchronization pipeline for a given repository.
/// Führt die Synchronisations-Pipeline für ein bestimmtes Repository aus.
pub async fn sync_single_repository(
repo_config: &RepoConfig,
app_config: &AppConfig,
@@ -47,7 +47,7 @@ pub async fn sync_single_repository(
LogLevel::Info,
);
// Resolve Gitea credentials
// Gitea-Zugangsdaten auflösen
let gitea_url = app_config.gitea_url.as_deref().unwrap_or_default();
let gitea_token = app_config.gitea_token.as_deref().unwrap_or_default();
let registry_owner = app_config.registry_owner.as_deref().unwrap_or_default();
@@ -155,12 +155,12 @@ pub async fn sync_single_repository(
continue;
}
// Download asset to temporary file
// Asset in temporäre Datei herunterladen
let temp_file_path = download_to_temp_file(&github_client, &asset.download_url, &asset.name)
.await
.with_context(|| format!("Failed to download asset '{}'", asset.name))?;
// Upload asset to all target distribution URLs
// Asset auf alle Ziel-Distributions-URLs hochladen
for url in &target_urls {
log(
"upload",
@@ -197,14 +197,14 @@ pub async fn sync_single_repository(
&format!("Failed to upload '{}' to {}: {:#}", asset.name, url, e),
LogLevel::Error,
);
// Cleanup temp file before returning error
// Temporäre Datei vor Fehlerrückgabe bereinigen
let _ = remove_file(&temp_file_path).await;
return Err(e);
}
}
}
// Cleanup temp file
// Temporäre Datei bereinigen
if let Err(e) = remove_file(&temp_file_path).await {
log(
"sync",
@@ -219,7 +219,7 @@ pub async fn sync_single_repository(
}
}
// Persist last synced tag if not dry-run
// Zuletzt synchronisierten Tag persistieren, wenn kein Dry-Run
if !options.dry_run {
if let Some(tag) = latest_synced_tag {
let name_copy = repo_name.to_string();
@@ -233,7 +233,7 @@ pub async fn sync_single_repository(
Ok(())
}
/// Downloads a GitHub release asset into a unique temporary file.
/// Lädt ein GitHub-Release-Asset in eine eindeutige temporäre Datei herunter.
async fn download_to_temp_file(
github_client: &GitHubClient,
download_url: &str,
+64
View File
@@ -0,0 +1,64 @@
/// Sanitizes a string by removing surrounding quotes (single or double).
///
/// # Arguments
///
/// * `input` - The string to sanitize.
///
/// # Returns
///
/// The sanitized string with surrounding quotes removed.
pub fn sanitize_string(input: &str) -> String {
let trimmed = input.trim();
if let Some(stripped) = trimmed.strip_prefix('"').and_then(|s| s.strip_suffix('"')) {
stripped.to_string()
} else if let Some(stripped) = trimmed.strip_prefix('\'').and_then(|s| s.strip_suffix('\'')) {
stripped.to_string()
} else {
trimmed.to_string()
}
}
/// Sanitizes environment variables by removing surrounding quotes.
///
/// # Arguments
///
/// * `env_vars` - A reference to a mutable map of environment variables.
pub fn sanitize_env_vars(env_vars: &mut std::collections::HashMap<String, String>) {
for (_key, value) in env_vars.iter_mut() {
*value = sanitize_string(value);
}
}
/// Bereinigt eine Repository-Eingabe (URL oder Kurzform), indem Query-Parameter, Fragmente,
/// URL-Schemata/Hosts (`github.com`, `www.github.com`, `git@github.com:`), `.git`-Endungen
/// und umgebende Schrägstriche entfernt werden.
pub fn clean_repo_input(input: &str) -> &str {
let mut trimmed = input.trim();
if (trimmed.starts_with('"') && trimmed.ends_with('"'))
|| (trimmed.starts_with('\'') && trimmed.ends_with('\''))
{
if trimmed.len() >= 2 {
trimmed = trimmed[1..trimmed.len() - 1].trim();
}
}
let without_query_or_fragment = match trimmed.find(|c| c == '?' || c == '#') {
Some(idx) => &trimmed[..idx],
None => trimmed,
};
let mut cleaned = without_query_or_fragment
.trim_start_matches("git@github.com:")
.trim_start_matches("https://")
.trim_start_matches("http://")
.trim_start_matches("www.github.com/")
.trim_start_matches("github.com/")
.trim_matches('/');
if let Some(stripped) = cleaned.strip_suffix(".git") {
cleaned = stripped.trim_matches('/');
}
cleaned
}
+146 -36
View File
@@ -1,53 +1,163 @@
use mirror_package::config::{AppConfig, RepoConfig};
use mirror_package::config::{AppConfig, RepoConfig, load_config};
use mirror_package::utils::{clean_repo_input, sanitize_env_vars, sanitize_string};
use std::collections::HashMap;
#[test]
fn test_sanitize_env_vars() {
let mut env_vars = HashMap::new();
env_vars.insert("GITEA_URL".to_string(), "\"https://gitea.example.com\"".to_string());
env_vars.insert("GITEA_TOKEN".to_string(), "\"token123\"".to_string());
env_vars.insert("REGISTRY_OWNER".to_string(), "\"owner\"".to_string());
env_vars.insert("GITHUB_TOKEN".to_string(), "\"github_token123\"".to_string());
sanitize_env_vars(&mut env_vars);
assert_eq!(env_vars.get("GITEA_URL").unwrap(), "https://gitea.example.com");
assert_eq!(env_vars.get("GITEA_TOKEN").unwrap(), "token123");
assert_eq!(env_vars.get("REGISTRY_OWNER").unwrap(), "owner");
assert_eq!(env_vars.get("GITHUB_TOKEN").unwrap(), "github_token123");
}
#[test]
fn test_load_config_with_env_vars() {
// This test relies on environment variables being set externally.
// To avoid `unsafe` blocks, we skip setting them programmatically.
// In a real test environment, set these variables before running the test:
// GITEA_URL="https://gitea.example.com"
// GITEA_TOKEN="token123"
// REGISTRY_OWNER="owner"
// GITHUB_TOKEN="github_token123"
let config = load_config();
// Only assert if the environment variables are set
if std::env::var("GITEA_URL").is_ok() {
assert_eq!(config.gitea_url, Some("https://gitea.example.com".to_string()));
assert_eq!(config.gitea_token, Some("token123".to_string()));
assert_eq!(config.registry_owner, Some("owner".to_string()));
assert_eq!(config.github_token, Some("github_token123".to_string()));
}
}
#[test]
fn test_sanitize_string() {
assert_eq!(sanitize_string("\"https://gitea.example.com\""), "https://gitea.example.com");
assert_eq!(sanitize_string("'https://gitea.example.com'"), "https://gitea.example.com");
assert_eq!(sanitize_string("\"token123\""), "token123");
assert_eq!(sanitize_string("'token123'"), "token123");
assert_eq!(sanitize_string("\"owner\""), "owner");
assert_eq!(sanitize_string("'owner'"), "owner");
assert_eq!(sanitize_string("\"github_token123\""), "github_token123");
assert_eq!(sanitize_string("'github_token123'"), "github_token123");
assert_eq!(sanitize_string("no_quotes"), "no_quotes");
assert_eq!(sanitize_string("\"single_quote\""), "single_quote");
assert_eq!(sanitize_string("'single_quote'"), "single_quote");
assert_eq!(sanitize_string("\"escaped_quote\""), "escaped_quote");
// Edge cases: single character inputs
assert_eq!(sanitize_string("\""), "\"");
assert_eq!(sanitize_string("'"), "'");
assert_eq!(sanitize_string("a"), "a");
// Edge cases: empty quotes and empty strings
assert_eq!(sanitize_string("\"\""), "");
assert_eq!(sanitize_string("''"), "");
assert_eq!(sanitize_string(""), "");
assert_eq!(sanitize_string(" "), "");
assert_eq!(sanitize_string(" 'hello' "), "hello");
assert_eq!(sanitize_string(" \"world\" "), "world");
}
#[test]
fn test_clean_repo_input() {
assert_eq!(clean_repo_input("owner/repo"), "owner/repo");
assert_eq!(clean_repo_input("https://github.com/owner/repo"), "owner/repo");
assert_eq!(clean_repo_input("http://github.com/owner/repo"), "owner/repo");
assert_eq!(clean_repo_input("github.com/owner/repo"), "owner/repo");
assert_eq!(clean_repo_input("https://github.com/owner/repo.git"), "owner/repo");
assert_eq!(clean_repo_input("https://github.com/owner/repo/"), "owner/repo");
assert_eq!(
clean_repo_input("https://github.com/raspberrypi/rpi-imager?tab=readme-ov-file"),
"raspberrypi/rpi-imager"
);
assert_eq!(
clean_repo_input("https://github.com/raspberrypi/rpi-imager/?tab=readme-ov-file"),
"raspberrypi/rpi-imager"
);
assert_eq!(
clean_repo_input("https://github.com/raspberrypi/rpi-imager.git?tab=readme-ov-file"),
"raspberrypi/rpi-imager"
);
assert_eq!(
clean_repo_input("https://github.com/raspberrypi/rpi-imager#readme"),
"raspberrypi/rpi-imager"
);
assert_eq!(
clean_repo_input("https://github.com/raspberrypi/rpi-imager?tab=readme-ov-file#install"),
"raspberrypi/rpi-imager"
);
assert_eq!(
clean_repo_input("raspberrypi/rpi-imager?tab=readme-ov-file"),
"raspberrypi/rpi-imager"
);
assert_eq!(
clean_repo_input("\"https://github.com/raspberrypi/rpi-imager?tab=readme-ov-file\""),
"raspberrypi/rpi-imager"
);
}
#[test]
fn test_normalize_repo_name() {
let input = "https://github.com/owner/repo.git";
let normalized = AppConfig::normalize_repo_name(input);
assert_eq!(normalized, "owner/repo");
let input_with_query = "https://github.com/raspberrypi/rpi-imager?tab=readme-ov-file";
assert_eq!(
AppConfig::normalize_repo_name("raspberrypi/rpi-imager"),
AppConfig::normalize_repo_name(input_with_query),
"raspberrypi/rpi-imager"
);
assert_eq!(
AppConfig::normalize_repo_name("https://github.com/raspberrypi/rpi-imager"),
"raspberrypi/rpi-imager"
);
assert_eq!(
AppConfig::normalize_repo_name("https://github.com/raspberrypi/rpi-imager.git"),
"raspberrypi/rpi-imager"
);
assert_eq!(
AppConfig::normalize_repo_name("Heroic-Games-Launcher/HeroicGamesLauncher/"),
"Heroic-Games-Launcher/HeroicGamesLauncher"
);
}
#[test]
fn test_repo_management() {
fn test_add_or_update_repo() {
let mut config = AppConfig::default();
config.add_or_update_repo(RepoConfig::new("owner/repo1", true));
config.add_or_update_repo(RepoConfig::new("owner/repo2", false));
assert_eq!(config.repositories.len(), 2);
assert!(config.find_repo("owner/repo1").is_some());
assert!(config.find_repo("https://github.com/owner/repo1").is_some());
assert!(config.remove_repo("owner/repo1"));
let repo = RepoConfig::new("https://github.com/owner/repo?tab=readme-ov-file", true);
config.add_or_update_repo(repo.clone());
assert_eq!(config.repositories.len(), 1);
assert!(config.find_repo("owner/repo1").is_none());
assert_eq!(config.repositories[0].name, "owner/repo");
assert_eq!(config.repositories[0].include_prereleases, true);
// Updating existing repo with a URL with query param
let repo_updated = RepoConfig::new("https://github.com/owner/repo#readme", false);
config.add_or_update_repo(repo_updated);
assert_eq!(config.repositories.len(), 1);
assert_eq!(config.repositories[0].name, "owner/repo");
assert_eq!(config.repositories[0].include_prereleases, false);
}
#[test]
fn test_serialization_roundtrip() {
let mut config = AppConfig {
gitea_url: Some("https://gitea.example.com".to_string()),
gitea_token: Some("secret_token".to_string()),
registry_owner: Some("my-org".to_string()),
github_token: Some("gh_pat".to_string()),
repositories: Vec::new(),
};
config.add_or_update_repo(RepoConfig::new("raspberrypi/rpi-imager", true));
fn test_remove_repo() {
let mut config = AppConfig::default();
let repo = RepoConfig::new("owner/repo", true);
config.add_or_update_repo(repo.clone());
assert!(config.remove_repo("https://github.com/owner/repo?tab=readme-ov-file"));
assert!(config.repositories.is_empty());
}
let json_str = serde_json::to_string(&config).unwrap();
let deserialized: AppConfig = serde_json::from_str(&json_str).unwrap();
#[test]
fn test_find_repo() {
let mut config = AppConfig::default();
let repo = RepoConfig::new("owner/repo", true);
config.add_or_update_repo(repo.clone());
assert_eq!(
config.find_repo("https://github.com/owner/repo?tab=readme-ov-file"),
Some(&RepoConfig::new("owner/repo", true))
);
}
assert_eq!(config, deserialized);
#[test]
fn test_update_last_synced_tag() {
let mut config = AppConfig::default();
let repo = RepoConfig::new("owner/repo", true);
config.add_or_update_repo(repo.clone());
config.update_last_synced_tag("owner/repo", "v1.0.0".to_string());
assert_eq!(config.find_repo("owner/repo").unwrap().last_synced_tag, Some("v1.0.0".to_string()));
}
+6 -6
View File
@@ -6,7 +6,7 @@ fn test_debian_routing() {
let base_url = "https://gitea.example.com";
let owner = "test-owner";
// Stable Debian goes to both stable and testing pools
// Stabile Debian-Pakete werden in stable und testing Pools hochgeladen
let stable_urls = get_target_upload_urls(base_url, owner, PackageType::Debian, false);
assert_eq!(
stable_urls,
@@ -16,7 +16,7 @@ fn test_debian_routing() {
]
);
// Pre-release Debian goes only to testing pool
// Pre-Release Debian-Pakete gehen nur in den testing Pool
let prerelease_urls = get_target_upload_urls(base_url, owner, PackageType::Debian, true);
assert_eq!(
prerelease_urls,
@@ -29,14 +29,14 @@ fn test_rpm_routing() {
let base_url = "https://gitea.example.com/";
let owner = "test-owner";
// Stable RPM
// Stabiles RPM
let stable_urls = get_target_upload_urls(base_url, owner, PackageType::Rpm, false);
assert_eq!(
stable_urls,
vec!["https://gitea.example.com/api/packages/test-owner/rpm/stable/upload"]
);
// Pre-release RPM
// Pre-Release RPM
let prerelease_urls = get_target_upload_urls(base_url, owner, PackageType::Rpm, true);
assert_eq!(
prerelease_urls,
@@ -49,14 +49,14 @@ fn test_arch_routing() {
let base_url = "https://gitea.example.com";
let owner = "test-owner";
// Stable Arch
// Stabiles Arch
let stable_urls = get_target_upload_urls(base_url, owner, PackageType::Arch, false);
assert_eq!(
stable_urls,
vec!["https://gitea.example.com/api/packages/test-owner/arch/stable"]
);
// Pre-release Arch
// Pre-Release Arch
let prerelease_urls = get_target_upload_urls(base_url, owner, PackageType::Arch, true);
assert_eq!(
prerelease_urls,
+34 -2
View File
@@ -1,4 +1,4 @@
use mirror_package::github::{parse_repo_owner_name, PackageType};
use mirror_package::github::{PackageType, parse_repo_owner_name};
#[test]
fn test_package_classification() {
@@ -40,7 +40,7 @@ fn test_package_classification() {
Some(PackageType::Arch)
);
// Non-package formats should be ignored
// Nicht unterstützte Paketformate sollten ignoriert werden
assert_eq!(PackageType::from_filename("rpi-imager-1.8.5.AppImage"), None);
assert_eq!(PackageType::from_filename("rpi-imager-1.8.5.dmg"), None);
assert_eq!(PackageType::from_filename("rpi-imager-1.8.5.exe"), None);
@@ -64,7 +64,39 @@ fn test_parse_repo_owner_name() {
parse_repo_owner_name("https://github.com/Heroic-Games-Launcher/HeroicGamesLauncher.git").unwrap(),
("Heroic-Games-Launcher", "HeroicGamesLauncher")
);
assert_eq!(
parse_repo_owner_name("https://github.com/raspberrypi/rpi-imager?tab=readme-ov-file").unwrap(),
("raspberrypi", "rpi-imager")
);
assert_eq!(
parse_repo_owner_name("https://github.com/raspberrypi/rpi-imager/?tab=readme-ov-file#install").unwrap(),
("raspberrypi", "rpi-imager")
);
assert_eq!(
parse_repo_owner_name("raspberrypi/rpi-imager?tab=readme-ov-file").unwrap(),
("raspberrypi", "rpi-imager")
);
assert!(parse_repo_owner_name("invalid_repo").is_err());
assert!(parse_repo_owner_name("invalid/repo/extra").is_err());
}
#[tokio::test]
async fn test_github_client_empty_token() {
use mirror_package::github::GitHubClient;
// Test that a client with an empty or whitespace token sanitizes it to None
let client = GitHubClient::new(Some("".to_string())).expect("Failed to create client with empty token");
assert_eq!(client.token(), None);
let client_whitespace = GitHubClient::new(Some(" ".to_string())).expect("Failed to create client with whitespace token");
assert_eq!(client_whitespace.token(), None);
// Test with a valid token
let client_with_token = GitHubClient::new(Some("valid_token".to_string())).expect("Failed to create client with valid token");
assert_eq!(client_with_token.token(), Some("valid_token"));
// Test with no token
let client_no_token = GitHubClient::new(None).expect("Failed to create client with no token");
assert_eq!(client_no_token.token(), None);
}