schalli/tessera-ctl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 17 Wiki Activity

feat(nextcloud-files): Hoch- und Herunterladen als Datenstrom, große Dateien in Stücken, ZIP

Browse Source 
- Hochladen ohne Pufferung: ganze Datei oder Chunked Upload v2 in 8-MiB-Stücken, Zusammenbau im Hintergrund mit abfragbarem Zustand
- Überschreibschutz (If-None-Match / Entity-Tag-Prüfung), Download mit Range, Content-Disposition und Schutzkopfzeilen von Tessera selbst
- Ordner- und Auswahl-ZIP; jeder Nextcloud-Fehler wird vor dem ersten Byte abgebildet, nie 401/403
- Browser-Hochlader mit Fortschritt, Wiederholung, Abbruch und Aufräumen

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Schalli
2026-10-08 18:30:02 +02:00
parent 8bee65c306
commit 4d4a0b31cb
15 changed files with 3431 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.planning/quick/261008-mzu-modul-nextcloud-dateien-eigenstaendiger-/e2e/e2e-transfer.sh
+216
View File
@@ -0,0 +1,216 @@
#!/usr/bin/env bash
# Aufgabe 4: Hoch- und Herunterladen gegen die Test-Nextcloud — Umlautdatei, Ueberschreibschutz,
# 30 MB in 8-MiB-Stuecken DURCH den Next.js-Proxy (/api-proxy), Download byteidentisch mit
# Range, Kopfzeilen, 413, 411, 404-Abbildung, ZIP (Ordner und Auswahl). Setzt einen laufenden
# Stack und nc-test-setup.sh voraus. Nur Testwerte; liest keine .env-Dateien.
set -euo pipefail
source "$(dirname "${BASH_SOURCE[0]}")/e2e-lib.sh"
trap 'rm -rf "$E2E_TMP"' EXIT
FILES="$API/modules/nextcloud-files"
VIA_WEB="$WEB/modules/nextcloud-files"
NC_DAV="http://localhost:18080/remote.php/dav"
ANNA="anna:User1-Pass-12345"
UMLAUT_NAME='Ärger & Ölpreis 100%.txt'
CHUNK=8388608
# enc <text> — vollstaendig prozentcodiert (jedes Zeichen ausser A-Za-z0-9._~-)
enc() { python3 -I -c 'import sys,urllib.parse;print(urllib.parse.quote(sys.argv[1], safe=""))' "$1"; }
# jget <datei> <ausdruck> — Wert aus einer JSON-Datei, z. B. 'd["mode"]'
jget() { python3 -I -c 'import json,sys;d=json.load(open(sys.argv[1]));print(eval(sys.argv[2]))' "$1" "$2"; }
ADMIN="$E2E_TMP/admin.jar"
e2e_wait_health
e2e_login "$ADMIN"
e2e_activate "$ADMIN"
e2e_set_address "$ADMIN"
code=$(e2e_connect_anna "$ADMIN")
e2e_expect 200 "$code" "anna verbinden"
F="/Tessera-E2E-T-$(date +%s)"
FNAME="${F#/}"
code=$(e2e_status "$ADMIN" POST "$FILES/folders" "{\"path\":\"$F\"}")
case "$code" in 200|201) ;; *) e2e_fail "Testordner anlegen -> $code" ;; esac
# --- 1. Kleine Datei mit Umlauten, Sonderzeichen und Prozent im Namen ------------------------------
printf 'Hallo Aerger\n' > "$E2E_TMP/small.txt"
SMALL_SIZE=$(wc -c < "$E2E_TMP/small.txt")
UPATH="$F/$UMLAUT_NAME"
code=$(e2e_status "$ADMIN" POST "$FILES/uploads" "{\"path\":\"$UPATH\",\"size\":$SMALL_SIZE}")
case "$code" in 200|201) ;; *) e2e_fail "Upload starten -> $code" ;; esac
e2e_contains "$E2E_TMP/body.out" '"mode":"single"' "kleine Datei: single"
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X PUT -H 'Content-Type: application/octet-stream' \
--data-binary "@$E2E_TMP/small.txt" "$FILES/uploads/file?path=$(enc "$UPATH")&size=$SMALL_SIZE&mtime=1700000000")
e2e_expect 200 "$code" "kleine Datei hochladen"
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -G --data-urlencode "path=$F" "$FILES/files")
e2e_expect 200 "$code" "Testordner auflisten"
e2e_contains "$E2E_TMP/body.out" "\"name\":\"$UMLAUT_NAME\"" "Umlautname exakt in der Liste"
ETAG=$(python3 -I -c 'import json,sys
for e in json.load(open(sys.argv[1]))["entries"]:
if e["name"]==sys.argv[2]: print(e["etag"]); print(e["mtime"], file=sys.stderr)' "$E2E_TMP/body.out" "$UMLAUT_NAME" 2> "$E2E_TMP/mtime.out")
[ "$(cat "$E2E_TMP/mtime.out")" = "2023-11-14T22:13:20.000Z" ] || e2e_fail "Aenderungszeit uebernommen: $(cat "$E2E_TMP/mtime.out")"
# --- 2. Ueberschreibschutz ------------------------------------------------------------------------------
code=$(e2e_status "$ADMIN" POST "$FILES/uploads" "{\"path\":\"$UPATH\",\"size\":$SMALL_SIZE}")
e2e_expect 409 "$code" "gleicher Name noch einmal"
e2e_contains "$E2E_TMP/body.out" '"code":"nameTaken"' "Namenskonflikt: nameTaken"
e2e_contains "$E2E_TMP/body.out" '"existing"' "Namenskonflikt: existing"
# direkter PUT ohne vorherigen Start darf ebenfalls nichts ueberschreiben (If-None-Match)
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X PUT -H 'Content-Type: application/octet-stream' \
--data-binary "@$E2E_TMP/small.txt" "$FILES/uploads/file?path=$(enc "$UPATH")&size=$SMALL_SIZE")
e2e_expect 409 "$code" "PUT auf vorhandenen Namen"
e2e_contains "$E2E_TMP/body.out" '"code":"nameTaken"' "PUT: nameTaken"
# Ersetzen mit der richtigen Version klappt, mit einer falschen nicht
printf 'Neuer Inhalt\n' > "$E2E_TMP/small2.txt"
SIZE2=$(wc -c < "$E2E_TMP/small2.txt")
code=$(e2e_status "$ADMIN" POST "$FILES/uploads" "{\"path\":\"$UPATH\",\"size\":$SIZE2,\"replaceEtag\":\"\\\"falsch\\\"\"}")
e2e_expect 409 "$code" "Ersetzen mit falscher Version"
e2e_contains "$E2E_TMP/body.out" '"code":"changedMeanwhile"' "falsche Version: changedMeanwhile"
ETAG_JSON=$(python3 -I -c 'import json,sys;print(json.dumps(sys.argv[1]))' "$ETAG")
code=$(e2e_status "$ADMIN" POST "$FILES/uploads" "{\"path\":\"$UPATH\",\"size\":$SIZE2,\"replaceEtag\":$ETAG_JSON}")
case "$code" in 200|201) ;; *) e2e_fail "Ersetzen mit richtiger Version -> $code" ;; esac
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X PUT -H 'Content-Type: application/octet-stream' \
--data-binary "@$E2E_TMP/small2.txt" "$FILES/uploads/file?path=$(enc "$UPATH")&size=$SIZE2&replaceEtag=$(enc "$ETAG")")
e2e_expect 200 "$code" "Ersetzen"
code=$(curl -s -o "$E2E_TMP/dl.out" -w '%{http_code}' -b "$ADMIN" "$FILES/download?path=$(enc "$UPATH")")
e2e_expect 200 "$code" "Umlautdatei laden"
cmp "$E2E_TMP/dl.out" "$E2E_TMP/small2.txt" || e2e_fail "ersetzte Datei hat nicht den neuen Inhalt"
# --- 3. 30 MB in 8-MiB-Stuecken durch den Next.js-Proxy ---------------------------------------------
head -c 30000000 /dev/urandom > "$E2E_TMP/big.bin"
BIG="$F/gross.bin"
BIG_SIZE=30000000
code=$(e2e_status "$ADMIN" POST "$VIA_WEB/uploads" "{\"path\":\"$BIG\",\"size\":$BIG_SIZE}")
case "$code" in 200|201) ;; *) e2e_fail "grosse Datei: Start -> $code" ;; esac
e2e_contains "$E2E_TMP/body.out" '"mode":"chunked"' "grosse Datei: chunked"
UP=$(jget "$E2E_TMP/body.out" 'd["uploadId"]')
[ "$(jget "$E2E_TMP/body.out" 'd["chunkSize"]')" = "$CHUNK" ] || e2e_fail "chunkSize ist nicht 8388608"
echo "$UP" | grep -Eq '^tessera-[0-9a-f-]{36}$' || e2e_fail "uploadId hat falsche Form: $UP"
mkdir "$E2E_TMP/chunks"
split -b "$CHUNK" -d -a 3 "$E2E_TMP/big.bin" "$E2E_TMP/chunks/c"
n=0
for part in "$E2E_TMP"/chunks/c*; do
n=$((n + 1))
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X PUT -H 'Content-Type: application/octet-stream' \
--data-binary "@$part" "$VIA_WEB/uploads/$UP/chunks/$n?path=$(enc "$BIG")&size=$BIG_SIZE")
e2e_expect 200 "$code" "Stueck $n durch den Proxy"
[ "$(jget "$E2E_TMP/body.out" 'd["received"]')" = "$(wc -c < "$part")" ] || e2e_fail "Stueck $n: received stimmt nicht"
done
[ "$n" = "4" ] || e2e_fail "30 MB sollten 4 Stuecke sein, waren $n"
code=$(e2e_status "$ADMIN" POST "$VIA_WEB/uploads/$UP/complete" "{\"path\":\"$BIG\",\"size\":$BIG_SIZE,\"mtime\":1700000000}")
case "$code" in
200) e2e_contains "$E2E_TMP/body.out" '"state":"done"' "complete: done" ;;
202)
for _ in $(seq 1 90); do
sleep 2
code=$(e2e_status "$ADMIN" GET "$VIA_WEB/uploads/$UP/state")
e2e_expect 200 "$code" "Zustand abfragen"
grep -q '"state":"done"' "$E2E_TMP/body.out" && break
grep -q '"state":"failed"' "$E2E_TMP/body.out" && e2e_fail "Zusammenbau fehlgeschlagen: $(cat "$E2E_TMP/body.out")"
done
grep -q '"state":"done"' "$E2E_TMP/body.out" || e2e_fail "Zusammenbau wurde nicht fertig"
;;
*) e2e_fail "complete -> $code" ;;
esac
code=$(e2e_status "$ADMIN" GET "$VIA_WEB/uploads/$UP/state")
e2e_expect 200 "$code" "Zustand nach dem Ende"
e2e_contains "$E2E_TMP/body.out" '"state":"done"' "Zustand done"
# Groesse und Mtime bei der Nextcloud selbst pruefen
nc_size=$(curl -s -u "$ANNA" -I "$NC_DAV/files/anna$F/gross.bin" | tr -d '\r' | awk -F': ' 'tolower($1)=="content-length"{print $2}')
[ "$nc_size" = "$BIG_SIZE" ] || e2e_fail "Nextcloud meldet $nc_size Byte statt $BIG_SIZE"
# Download durch den Proxy, byteidentisch; Kopfzeilen
code=$(curl -s -D "$E2E_TMP/dl.hdr" -o "$E2E_TMP/big.dl" -w '%{http_code}' -b "$ADMIN" "$VIA_WEB/download?path=$(enc "$BIG")")
e2e_expect 200 "$code" "grosse Datei laden (Proxy)"
cmp "$E2E_TMP/big.bin" "$E2E_TMP/big.dl" || e2e_fail "heruntergeladene Datei weicht ab"
grep -qi '^content-disposition: attachment; filename="gross.bin"' "$E2E_TMP/dl.hdr" || e2e_fail "content-disposition fehlt/falsch: $(grep -i content-disposition "$E2E_TMP/dl.hdr")"
grep -qi '^x-content-type-options: nosniff' "$E2E_TMP/dl.hdr" || e2e_fail "nosniff fehlt"
grep -qi '^content-security-policy: default-src .none.; sandbox' "$E2E_TMP/dl.hdr" || e2e_fail "CSP sandbox fehlt"
if grep -qi '^set-cookie' "$E2E_TMP/dl.hdr"; then e2e_fail "set-cookie erreichte den Browser"; fi
# Umlautname im Content-Disposition
curl -s -D "$E2E_TMP/dl2.hdr" -o /dev/null -b "$ADMIN" "$FILES/download?path=$(enc "$UPATH")"
grep -qi "^content-disposition: attachment; filename=\"_rger & _lpreis 100_.txt\"; filename\\*=UTF-8''%C3%84rger%20%26%20%C3%96lpreis%20100%25.txt" "$E2E_TMP/dl2.hdr" \
|| e2e_fail "content-disposition mit Umlauten: $(grep -i content-disposition "$E2E_TMP/dl2.hdr")"
# Range
code=$(curl -s -D "$E2E_TMP/range.hdr" -o "$E2E_TMP/range.out" -w '%{http_code}' -b "$ADMIN" -H 'Range: bytes=0-99' "$VIA_WEB/download?path=$(enc "$BIG")")
e2e_expect 206 "$code" "Range"
[ "$(wc -c < "$E2E_TMP/range.out")" = "100" ] || e2e_fail "Range: nicht 100 Byte"
head -c 100 "$E2E_TMP/big.bin" | cmp - "$E2E_TMP/range.out" || e2e_fail "Range: Inhalt weicht ab"
grep -qi "^content-range: bytes 0-99/$BIG_SIZE" "$E2E_TMP/range.hdr" || e2e_fail "content-range fehlt: $(grep -i content-range "$E2E_TMP/range.hdr")"
code=$(curl -s -o "$E2E_TMP/range2.out" -w '%{http_code}' -b "$ADMIN" -H "Range: bytes=$((BIG_SIZE - 10))-" "$FILES/download?path=$(enc "$BIG")")
e2e_expect 206 "$code" "Range bis zum Ende"
tail -c 10 "$E2E_TMP/big.bin" | cmp - "$E2E_TMP/range2.out" || e2e_fail "Range bis zum Ende: Inhalt weicht ab"
# --- 4. Grenzen: 413, 411 --------------------------------------------------------------------------------
head -c 9000000 /dev/zero > "$E2E_TMP/nine.bin"
FAKE_UP="tessera-00000000-0000-0000-0000-000000000000"
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X PUT -H 'Content-Type: application/octet-stream' \
--data-binary "@$E2E_TMP/nine.bin" "$VIA_WEB/uploads/$FAKE_UP/chunks/1?path=$(enc "$F/x.bin")&size=30000000")
e2e_expect 413 "$code" "9-MB-Stueck"
e2e_contains "$E2E_TMP/body.out" '"code":"chunkTooLarge"' "9-MB-Stueck: chunkTooLarge"
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X PUT -H 'Transfer-Encoding: chunked' -H 'Content-Type: application/octet-stream' \
--data-binary "@$E2E_TMP/small.txt" "$FILES/uploads/$FAKE_UP/chunks/1?path=$(enc "$F/x.bin")&size=30000000")
e2e_expect 411 "$code" "PUT ohne content-length"
e2e_contains "$E2E_TMP/body.out" '"code":"lengthRequired"' "ohne content-length: lengthRequired"
code=$(e2e_status "$ADMIN" POST "$FILES/uploads" "{\"path\":\"$F/riesig.bin\",\"size\":99999999999999}")
e2e_expect 413 "$code" "zu grosse Datei"
e2e_contains "$E2E_TMP/body.out" '"code":"fileTooLarge"' "zu gross: fileTooLarge"
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X PUT -H 'Content-Type: application/octet-stream' \
--data-binary "@$E2E_TMP/small.txt" "$FILES/uploads/tessera-..%2Fx/chunks/1?path=$(enc "$F/x.bin")&size=30000000")
e2e_expect 400 "$code" "ungueltige Upload-Kennung"
# --- 5. Fehlerabbildung: nie 401/403 -----------------------------------------------------------------------
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" "$FILES/download?path=$(enc "$F/gibt-es-nicht.bin")")
e2e_expect 404 "$code" "Download einer fehlenden Datei"
e2e_contains "$E2E_TMP/body.out" '"code":"notFound"' "fehlende Datei: notFound"
# --- 6. ZIP -----------------------------------------------------------------------------------------------------
code=$(curl -s -D "$E2E_TMP/zip.hdr" -o "$E2E_TMP/folder.zip" -w '%{http_code}' -b "$ADMIN" "$VIA_WEB/download?path=$(enc "$F")&zip=1")
e2e_expect 200 "$code" "Ordner als ZIP"
[ "$(head -c 2 "$E2E_TMP/folder.zip")" = "PK" ] || e2e_fail "Ordner-ZIP beginnt nicht mit PK"
grep -qi "^content-disposition: attachment; filename=\"$FNAME.zip\"" "$E2E_TMP/zip.hdr" || e2e_fail "ZIP-Name: $(grep -i content-disposition "$E2E_TMP/zip.hdr")"
unzip -l "$E2E_TMP/folder.zip" | grep -q 'gross.bin' || e2e_fail "Ordner-ZIP enthaelt gross.bin nicht"
code=$(curl -s -D "$E2E_TMP/zip2.hdr" -o "$E2E_TMP/sel.zip" -w '%{http_code}' -b "$ADMIN" -G --data-urlencode "dir=$F" --data-urlencode "name=$UMLAUT_NAME" "$FILES/download/zip")
e2e_expect 200 "$code" "Auswahl als ZIP"
[ "$(head -c 2 "$E2E_TMP/sel.zip")" = "PK" ] || e2e_fail "Auswahl-ZIP beginnt nicht mit PK"
unzip -Z1 "$E2E_TMP/sel.zip" > "$E2E_TMP/sel.list"
[ "$(wc -l < "$E2E_TMP/sel.list")" = "1" ] || e2e_fail "Auswahl-ZIP enthaelt nicht genau einen Eintrag: $(cat "$E2E_TMP/sel.list")"
[ "$(cat "$E2E_TMP/sel.list")" = "$UMLAUT_NAME" ] || e2e_fail "Auswahl-ZIP: falscher Eintrag: $(cat "$E2E_TMP/sel.list")"
# zwei Namen (wiederholter Schluessel)
code=$(curl -s -o "$E2E_TMP/sel2.zip" -w '%{http_code}' -b "$ADMIN" -G --data-urlencode "dir=$F" --data-urlencode "name=$UMLAUT_NAME" --data-urlencode "name=gross.bin" "$FILES/download/zip")
e2e_expect 200 "$code" "Auswahl mit zwei Namen"
[ "$(unzip -Z1 "$E2E_TMP/sel2.zip" | wc -l)" = "2" ] || e2e_fail "Auswahl-ZIP mit zwei Namen hat nicht zwei Eintraege"
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -G --data-urlencode "dir=$F" --data-urlencode "name=.." "$FILES/download/zip")
e2e_expect 400 "$code" "ZIP mit Name .."
e2e_contains "$E2E_TMP/body.out" '"code":"invalidPath"' "ZIP-Name ..: invalidPath"
# --- 7. Abbrechen raeumt auf -----------------------------------------------------------------------------------------
code=$(e2e_status "$ADMIN" POST "$FILES/uploads" "{\"path\":\"$F/abbruch.bin\",\"size\":$BIG_SIZE}")
case "$code" in 200|201) ;; *) e2e_fail "Abbruch-Upload starten -> $code" ;; esac
UP2=$(jget "$E2E_TMP/body.out" 'd["uploadId"]')
code=$(curl -s -o /dev/null -w '%{http_code}' -u "$ANNA" -X PROPFIND -H 'Depth: 0' "$NC_DAV/uploads/anna/$UP2")
e2e_expect 207 "$code" "Upload-Ordner liegt bei Nextcloud"
code=$(e2e_status "$ADMIN" DELETE "$FILES/uploads/$UP2")
e2e_expect 200 "$code" "Upload abbrechen"
code=$(curl -s -o /dev/null -w '%{http_code}' -u "$ANNA" -X PROPFIND -H 'Depth: 0' "$NC_DAV/uploads/anna/$UP2")
e2e_expect 404 "$code" "Upload-Ordner nach dem Abbrechen"
code=$(e2e_status "$ADMIN" GET "$FILES/uploads/$UP2/state")
e2e_expect 404 "$code" "Zustand eines unbekannten Uploads"
# --- 8. Zwei Benutzer teilen keinen Zustand: ohne Anmeldung gar nichts ------------------------------------------------
code=$(curl -s -o /dev/null -w '%{http_code}' "$FILES/download?path=$(enc "$UPATH")")
e2e_expect 401 "$code" "Download ohne Tessera-Anmeldung"
# --- Aufraeumen ---------------------------------------------------------------------------------------------------------------
code=$(curl -s -o "$E2E_TMP/body.out" -w '%{http_code}' -b "$ADMIN" -X DELETE -G --data-urlencode "path=$F" "$FILES/files")
e2e_expect 200 "$code" "Testordner loeschen"
code=$(e2e_status "$ADMIN" DELETE "$FILES/connect")
e2e_expect 200 "$code" "anna trennen"
echo "e2e transfer ok"