tools/reassemble.sh

#!/bin/bash
#
# Reassemble split files after downloading from HuggingFace.
# Finds all *.00.part files, concatenates the parts, and removes them.
#
# Usage: ./reassemble.sh
#
set -euo pipefail
cd "$(dirname "$0")"

echo "=== Reassembling split files ==="

# Find all first parts (*.00.part) to identify split files
FIRST_PARTS=()
while IFS= read -r -d '' f; do
    FIRST_PARTS+=("$f")
done < <(find . -type f -name '*.00.part' -print0)

if [ ${#FIRST_PARTS[@]} -eq 0 ]; then
    echo "No split files found. Nothing to reassemble."
    exit 0
fi

echo "Found ${#FIRST_PARTS[@]} split files to reassemble:"
for f in "${FIRST_PARTS[@]}"; do
    # Original name: remove the .00.part suffix
    original="${f%.00.part}"
    parts=$(ls "${original}".*.part 2>/dev/null | wc -l)
    total=$(du -ch "${original}".*.part 2>/dev/null | tail -1 | cut -f1)
    echo "  $original ($parts parts, $total total)"
done
echo ""

for f in "${FIRST_PARTS[@]}"; do
    original="${f%.00.part}"
    echo "[REASSEMBLE] $original"
    cat "${original}".*.part > "$original"
    echo "  Assembled: $(du -h "$original" | cut -f1)"
    rm "${original}".*.part
    echo "  Removed parts."
    echo ""
done

# Verify against manifest if available
if [ -f MANIFEST.txt ]; then
    echo "=== Verifying against MANIFEST.txt ==="
    errors=0
    while read -r expected_size filepath; do
        if [ ! -f "$filepath" ]; then
            # Skip part files listed in manifest (already removed)
            [[ "$filepath" == *.part ]] && continue
            echo "  MISSING: $filepath"
            errors=$((errors + 1))
            continue
        fi
        actual_size=$(stat -c%s "$filepath")
        if [ "$actual_size" != "$expected_size" ]; then
            echo "  SIZE MISMATCH: $filepath (expected $expected_size, got $actual_size)"
            errors=$((errors + 1))
        fi
    done < MANIFEST.txt
    if [ $errors -eq 0 ]; then
        echo "  All files verified OK."
    else
        echo "  WARNING: $errors verification errors."
    fi
fi

echo ""
echo "=== Done ==="