Refactor and enhance SplitInChunks.sh script
Removed unused variables and streamlined logic for better readability and maintainability. Added dependency checks, improved path validation, dynamic parallelism based on CPU cores, and detailed progress/error reporting. Ensured robust cleanup and verification mechanisms for accurate chunking and file transfers.
This commit is contained in:
+196
-115
@@ -4,8 +4,6 @@ set -euo pipefail
|
||||
|
||||
# === Konfiguration ===
|
||||
CHUNK_SIZE=4000
|
||||
MAX_JOBS=8 # Parallelitätsgrad (an CPU-Kerne anpassen)
|
||||
VERIFY_AFTER_COPY=false # Nachkopier-Prüfung aktivieren
|
||||
|
||||
# === Argumentverarbeitung ===
|
||||
if [[ $# -lt 2 ]]; then
|
||||
@@ -17,123 +15,206 @@ fi
|
||||
SOURCE_DIR="$1"
|
||||
DEST_BASE="$2"
|
||||
|
||||
# === Validierung ===
|
||||
if [[ ! -d "$SOURCE_DIR" ]]; then
|
||||
echo "Error: Source directory '$SOURCE_DIR' does not exist" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$DEST_BASE"
|
||||
|
||||
# === Temporäre Dateien ===
|
||||
FILE_LIST=$(mktemp)
|
||||
trap 'rm -f "$FILE_LIST" /tmp/part_* 2>/dev/null' EXIT
|
||||
|
||||
# === Schritt 1: Dateiliste erstellen ===
|
||||
echo "[INFO] Scanning source directory..."
|
||||
find "$SOURCE_DIR" -maxdepth 1 -type f -print0 | sort -z > "$FILE_LIST"
|
||||
TOTAL_FILES=$(grep -cz . "$FILE_LIST")
|
||||
echo "[INFO] Found $TOTAL_FILES files"
|
||||
|
||||
# === Schritt 2: Chunk-Einteilung berechnen ===
|
||||
NUM_CHUNKS=$(( (TOTAL_FILES + CHUNK_SIZE - 1) / CHUNK_SIZE ))
|
||||
echo "[INFO] Will split into $NUM_CHUNKS chunks (~$(( TOTAL_FILES / NUM_CHUNKS )) files per chunk)"
|
||||
|
||||
# === Schritt 3: Teildateien generieren ===
|
||||
echo "[INFO] Creating chunk index files..."
|
||||
split -z -l $CHUNK_SIZE -d "$FILE_LIST" "/tmp/part_"
|
||||
|
||||
# Index-Dateien umbenennen für Lesbarkeit
|
||||
for idx in $(seq 0 $(( NUM_CHUNKS - 1 ))); do
|
||||
old_name="/tmp/part_$(printf '%02d' $idx)"
|
||||
if [[ -f "$old_name" ]] && [[ "$idx" != "0" ]]; then
|
||||
mv "$old_name" "${old_name}.idx"
|
||||
elif [[ -f "/tmp/part_aa" ]]; then
|
||||
mv "/tmp/part_aa" "/tmp/part_0.idx"
|
||||
fi
|
||||
done
|
||||
|
||||
# === Schritt 4: Paralleles rsync starten ===
|
||||
echo "[INFO] Starting parallel copy with $MAX_JOBS workers..."
|
||||
|
||||
declare -A PID_MAP
|
||||
|
||||
copy_chunk() {
|
||||
local chunk_idx=$1
|
||||
local idx_file="/tmp/part_${chunk_idx}.idx"
|
||||
local target_dir="${DEST_BASE}/chunk_$(printf '%03d' $((chunk_idx + 1)))"
|
||||
# === Schritt 0: Abhängigkeiten prüfen ===
|
||||
check_dependencies() {
|
||||
echo "[CHECK] Prüfe auf erforderliche Programme..."
|
||||
|
||||
mkdir -p "$target_dir"
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
filename=$(basename "$file")
|
||||
rsync -a --info=progress2 "$file" "${target_dir}/${filename}" >/dev/null 2>&1 || \
|
||||
echo "[WARN] Failed: $filename" >&2
|
||||
done < "$idx_file"
|
||||
|
||||
if $VERIFY_AFTER_COPY; then
|
||||
while IFS= read -r -d '' src_file; do
|
||||
base=$(basename "$src_file")
|
||||
dst_file="${target_dir}/${base}"
|
||||
if ! diff -q "$src_file" "$dst_file" >/dev/null 2>&1; then
|
||||
echo "[ERROR] Verification failed: $base" >&2
|
||||
return 1
|
||||
fi
|
||||
done < "$idx_file"
|
||||
# Nur rsync explizit prüfen (coreutils/findutils/procps sind fast immer da)
|
||||
if ! command -v rsync &>/dev/null; then
|
||||
echo ""
|
||||
echo "❌ rsync nicht gefunden!"
|
||||
echo ""
|
||||
echo "Installieren:"
|
||||
echo " sudo apt update && sudo apt install rsync"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[OK] Chunk $((chunk_idx + 1)) complete"
|
||||
if ! command -v find &>/dev/null; then
|
||||
echo ""
|
||||
echo "❌ find nicht gefunden!"
|
||||
echo ""
|
||||
echo "Installieren:"
|
||||
echo " sudo apt update && sudo apt install findutils"
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf " ✓ rsync verfügbar\n"
|
||||
printf " ✓ find verfügbar\n\n"
|
||||
}
|
||||
|
||||
# Jobs starten mit Job-Controlle
|
||||
active_jobs=0
|
||||
chunk_counter=0
|
||||
|
||||
for idx_file in /tmp/*.idx; do
|
||||
[[ -f "$idx_file" ]] || continue
|
||||
|
||||
((chunk_counter++))
|
||||
copy_chunk $((chunk_counter - 1)) &
|
||||
PID_MAP[$chunk_counter]=$!
|
||||
((active_jobs++))
|
||||
|
||||
# Begrenzung der gleichzeitigen Prozesse
|
||||
while [[ $active_jobs -ge $MAX_JOBS ]]; do
|
||||
wait -n || true
|
||||
((active_jobs--))
|
||||
done
|
||||
done
|
||||
|
||||
# Auf alle Rest-Jobs warten
|
||||
echo "[INFO] Waiting for remaining jobs..."
|
||||
wait
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "FERTIG!"
|
||||
echo "------------------------------------------"
|
||||
echo "Quellverzeichnis: $SOURCE_DIR"
|
||||
echo "Zielbasisverzeichnis: $DEST_BASE"
|
||||
echo "Gesamtanzahl Dateien: $TOTAL_FILES"
|
||||
echo "Chunks erstellt: $chunk_counter"
|
||||
echo "Chunks pro File: $CHUNK_SIZE"
|
||||
echo "Parallelität: $MAX_JOBS Jobs"
|
||||
echo "=========================================="
|
||||
|
||||
# Optional: Chunk-Zählung verifizieren
|
||||
echo ""
|
||||
echo "[VERIFICATION] Counting final files..."
|
||||
final_count=0
|
||||
for chunk in "${DEST_BASE}"/chunk_*; do
|
||||
if [[ -d "$chunk" ]]; then
|
||||
count=$(find "$chunk" -maxdepth 1 -type f | wc -l)
|
||||
echo " $(basename "$chunk"): $count Dateien"
|
||||
((final_count += count))
|
||||
# === Validierung ===
|
||||
validate_paths() {
|
||||
if [[ ! -d "$SOURCE_DIR" ]]; then
|
||||
echo "Error: Source directory '$SOURCE_DIR' does not exist" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$DEST_BASE" != /* ]]; then
|
||||
DEST_BASE="$(pwd)/$DEST_BASE"
|
||||
fi
|
||||
|
||||
mkdir -p "$DEST_BASE" || {
|
||||
echo "Error: Cannot create destination '$DEST_BASE'" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
case "$DEST_BASE" in
|
||||
"$SOURCE_DIR"/*)
|
||||
echo "Error: Destination must not be subdirectory of source" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ ! -w "$DEST_BASE" ]]; then
|
||||
echo "Error: No write permission on destination '$DEST_BASE'" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [[ $final_count -eq $TOTAL_FILES ]]; then
|
||||
echo "[✓] Verifikation erfolgreich: $final_count/$TOTAL_FILES Dateien übertragen"
|
||||
else
|
||||
echo "[✗] WARNUNG: Diskrepanz detected ($final_count vs $TOTAL_FILES)"
|
||||
fi
|
||||
# === Hauptlogik ===
|
||||
main() {
|
||||
check_dependencies
|
||||
validate_paths
|
||||
|
||||
CPU_CORES=$(nproc 2>/dev/null || grep -c '^processor' /proc/cpuinfo 2>/dev/null || echo 4)
|
||||
MAX_JOBS=$(( CPU_CORES > 8 ? 8 : CPU_CORES ))
|
||||
echo "[INFO] CPU-Kerne: $CPU_CORES | Parallelität: $MAX_JOBS Jobs"
|
||||
echo ""
|
||||
|
||||
WORK_DIR=$(mktemp -d)
|
||||
LIST_FILE="${WORK_DIR}/filelist.null"
|
||||
trap 'rm -rf "$WORK_DIR"' EXIT
|
||||
|
||||
# === Schritt 1: Dateien ermitteln ===
|
||||
echo "[STEP 1] Durchsuche Quellverzeichnis..."
|
||||
find "$SOURCE_DIR" -maxdepth 1 -type f -print0 > "$LIST_FILE"
|
||||
|
||||
TOTAL_FILES=$(tr -cd '\0' < "$LIST_FILE" | wc -c)
|
||||
|
||||
if [[ "$TOTAL_FILES" -eq 0 ]]; then
|
||||
echo "[WARN] Keine Dateien in '$SOURCE_DIR' gefunden"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "[OK] Gefundene Dateien: $TOTAL_FILES"
|
||||
echo ""
|
||||
|
||||
NUM_CHUNKS=$(( (TOTAL_FILES + CHUNK_SIZE - 1) / CHUNK_SIZE ))
|
||||
|
||||
echo "[INFO] Aufteilung: $NUM_CHUNKS Chunks (bis max. $CHUNK_SIZE pro Chunk)"
|
||||
echo ""
|
||||
|
||||
# === Schritt 2: Paralleles Kopieren ===
|
||||
echo "[STEP 2] Starte paralleles Kopieren..."
|
||||
echo "---"
|
||||
|
||||
declare -a JOB_PIDS
|
||||
ACTIVE_JOBS=0
|
||||
|
||||
for ((chunk_num = 0; chunk_num < NUM_CHUNKS; chunk_num++)); do
|
||||
START_IDX=$(( chunk_num * CHUNK_SIZE ))
|
||||
END_IDX=$(( START_IDX + CHUNK_SIZE ))
|
||||
|
||||
TARGET_DIR="${DEST_BASE}/chunk_$(printf '%03d' $((chunk_num + 1)))"
|
||||
mkdir -p "$TARGET_DIR"
|
||||
|
||||
(
|
||||
local copied=0
|
||||
local failed=0
|
||||
local skipped=0
|
||||
current_idx=0
|
||||
|
||||
while IFS= read -r -d '' filepath; do
|
||||
if [[ $current_idx -ge $START_IDX ]] && [[ $current_idx -lt $END_IDX ]]; then
|
||||
|
||||
filename=$(basename -- "$filepath")
|
||||
DST_FILE="${TARGET_DIR}/${filename}"
|
||||
|
||||
if [[ -f "$DST_FILE" ]]; then
|
||||
((skipped++)) || true
|
||||
elif rsync -a --quiet -- "$filepath" "$DST_FILE" 2>/dev/null; then
|
||||
((copied++)) || true
|
||||
else
|
||||
echo "[FAIL] $(basename -- "$filepath")" >&2
|
||||
((failed++)) || true
|
||||
fi
|
||||
fi
|
||||
|
||||
((current_idx++)) || true
|
||||
|
||||
done < "$LIST_FILE"
|
||||
|
||||
echo "${copied}:${skipped}:${failed}" > "${WORK_DIR}/status_${chunk_num}.txt"
|
||||
|
||||
) &
|
||||
|
||||
JOB_PIDS+=($!)
|
||||
((ACTIVE_JOBS++)) || true
|
||||
|
||||
if [[ $ACTIVE_JOBS -ge $MAX_JOBS ]]; then
|
||||
wait -n "${JOB_PIDS[-1]}" 2>/dev/null || true
|
||||
unset 'JOB_PIDS[-1]'
|
||||
((ACTIVE_JOBS--)) || true
|
||||
fi
|
||||
|
||||
if [[ $(( (chunk_num + 1) % 5 )) -eq 0 ]]; then
|
||||
echo "[PROGRESS] Chunk(s) bis $((chunk_num + 1)) gestartet..."
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[INFO] Warte auf alle Jobs..."
|
||||
wait "${JOB_PIDS[@]}" 2>/dev/null || true
|
||||
echo ""
|
||||
|
||||
# === STEP 3: Zusammenfassung ===
|
||||
echo "=========================================="
|
||||
echo "FERTIG!"
|
||||
echo "------------------------------------------"
|
||||
printf "%-25s %s\n" "Quellverzeichnis:" "$SOURCE_DIR"
|
||||
printf "%-25s %s\n" "Zielbasisverzeichnis:" "$DEST_BASE"
|
||||
printf "%-25s %d\n" "Gesamtanzahl Dateien:" "$TOTAL_FILES"
|
||||
printf "%-25s %d\n" "Chunks erstellt:" "$NUM_CHUNKS"
|
||||
printf "%-25s %d\n" "Maximal pro Chunk:" "$CHUNK_SIZE"
|
||||
printf "%-25s %d\n" "Parallelität:" "$MAX_JOBS Jobs"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
|
||||
# === Verifikation ===
|
||||
echo "[VERIFICATION] Finale Zählung..."
|
||||
FINAL_COUNT=0
|
||||
ERRORS=0
|
||||
|
||||
for ((i = 0; i < NUM_CHUNKS; i++)); do
|
||||
STATUS_FILE="${WORK_DIR}/status_${i}.txt"
|
||||
CHUNK_DIR="${DEST_BASE}/chunk_$(printf '%03d' $((i + 1)))"
|
||||
|
||||
if [[ -f "$STATUS_FILE" ]]; then
|
||||
IFS=':' read -r COPIED SKIPPED FAILED < "$STATUS_FILE"
|
||||
ACTUAL_COUNT=$(find "$CHUNK_DIR" -maxdepth 1 -type f 2>/dev/null | wc -l)
|
||||
|
||||
ERROR_MSG=""
|
||||
if [[ $FAILED -gt 0 ]]; then
|
||||
ERROR_MSG=" [⚠ FEHLER: $FAILED]"
|
||||
((ERRORS += FAILED)) || true
|
||||
fi
|
||||
|
||||
printf " Chunk_%03d: %-6d Dateien%s\n" "$((i + 1))" "$ACTUAL_COUNT" "$ERROR_MSG"
|
||||
((FINAL_COUNT += ACTUAL_COUNT)) || true
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
|
||||
if [[ $FINAL_COUNT -eq $TOTAL_FILES ]] && [[ $ERRORS -eq 0 ]]; then
|
||||
echo "✓ SUCCESS: Alle $FINAL_COUNT Dateien übertragen"
|
||||
return 0
|
||||
else
|
||||
echo "! WARNUNG: $((TOTAL_FILES - FINAL_COUNT)) Dateien nicht erfasst"
|
||||
[[ $ERRORS -gt 0 ]] && echo " Fehler: $ERRORS"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user