Speed up and stabilize local file upload

client/src/App.vue

@@ -470,39 +470,89 @@ export default {
     },
     async uploadLocalFile() {
       const CHUNK_SIZE = 3 * 1024 * 1024;
+      const MAX_CHUNK_RETRIES = 20;
+      const CHUNK_TIMEOUT_MS = 20000;
+      const RETRY_BACKOFF_MS = 2000;
+      const RETRY_BACKOFF_MAX_MS = 30000;
       let self = this;
-      var reader = new FileReader();
       var offset = 0;
       var filesize = this.file.size;
 
-      reader.onload = function () {
-        var result = reader.result;
-        var chunk = result;
+      // Chunks are posted as raw binary Blob slices, not base64-encoded
+      // inside a JSON body - base64 inflates the wire size by ~33%,
+      // which mattered a lot on the WiFi-constrained upload path. This
+      // also drops the FileReader/readAsDataURL round trip entirely.
+      //
+      // Chunks are sent one at a time, not pipelined. Concurrent chunks
+      // were tried and reverted - this board's USB hub has WiFi and
+      // storage sharing a single Transaction Translator (confirmed from
+      // the hub's datasheet), so simultaneous network-receive and
+      // disk-write transactions destabilize the connection rather than
+      // just being slower.
+      //
+      // This link also has occasional multi-minute dead spells (no
+      // error, no response, ever - axios has no default timeout for
+      // that case) that look like genuine WiFi/AP hiccups rather than
+      // one-off blips. Without an explicit timeout and a generous retry
+      // budget, one bad stretch used to freeze the whole upload forever
+      // with no feedback - see #61. Retries use exponential backoff
+      // (capped) so a real multi-minute dropout can be ridden out
+      // instead of giving up after a few seconds.
+      function sendChunk(retriesLeft = MAX_CHUNK_RETRIES) {
+        var slice = self.file.slice(offset, offset + CHUNK_SIZE);
         axios
-          .post(`/api/upload_chunk`, {
-            chunk: chunk,
+          .post(`/api/upload_chunk`, slice, {
+            headers: { "Content-Type": "application/octet-stream" },
+            timeout: CHUNK_TIMEOUT_MS,
           })
           .then(function (response) {
             const status = response.data;
             if (status.success && self.state == "UPLOADING") {
               offset += CHUNK_SIZE;
-              if (offset <= filesize) {
-                var slice = self.file.slice(offset, offset + CHUNK_SIZE);
-                reader.readAsDataURL(slice);
+              if (offset < filesize) {
+                sendChunk();
               } else {
-                offset = filesize;
                 self.apiCall("upload_finish");
               }
             } else {
               self.apiCall("upload_cancel");
             }
+          })
+          .catch(function (error) {
+            if (self.state != "UPLOADING") {
+              return;
+            }
+            if (retriesLeft > 0) {
+              const attempt = MAX_CHUNK_RETRIES - retriesLeft;
+              const backoff = Math.min(
+                RETRY_BACKOFF_MS * Math.pow(2, attempt),
+                RETRY_BACKOFF_MAX_MS
+              );
+              console.log(
+                "upload_chunk failed, retrying in " +
+                  backoff +
+                  "ms (" +
+                  retriesLeft +
+                  " left): " +
+                  error
+              );
+              setTimeout(function () {
+                sendChunk(retriesLeft - 1);
+              }, backoff);
+            } else {
+              self.$waveui.notify(
+                "Upload failed after repeated retries: " + error,
+                "error",
+                0
+              );
+              self.apiCall("upload_cancel");
+            }
           });
-      };
+      }
 
       if (this.file) {
-        var slice = this.file.slice(offset, offset + CHUNK_SIZE);
-        reader.readAsDataURL(slice);
         self.fileName = this.file.name;
+        sendChunk();
       }
     },
     onMagicButtonClick() {

reflash/server.go

@@ -578,7 +578,11 @@ func uploadStart(w http.ResponseWriter, r *http.Request) {
 	timeStart = time.Now()
 	logInfo("Starting upload at " + timeStart.Format("15:04:05"))
 	logInfo("Filename: " + state.Filename)
-	os.Create(images_folder + "/" + state.Filename)
+	f, err := os.Create(images_folder + "/" + state.Filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+	state.File = f
 
 	sendResponse(w, nil)
 }
@@ -670,29 +674,39 @@ func uploadMagicFinish(w http.ResponseWriter, r *http.Request) {
 }
 
 func uploadChunk(w http.ResponseWriter, r *http.Request) {
-	var chunk *Chunk = &Chunk{}
-	reqBody, _ := io.ReadAll(r.Body)
-	json.Unmarshal(reqBody, &chunk)
-
-	decoded, err := base64.StdEncoding.DecodeString(chunk.Encoded[37:len(chunk.Encoded)])
-
-	path := images_folder + "/" + state.Filename
-
 	if state.State == CANCELLED {
 		response := map[string]bool{"success": false}
 		json.NewEncoder(w).Encode(response)
 		return
 	}
 
-	f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644)
+	// The client posts the chunk as a raw binary body (not base64/JSON -
+	// that inflated the wire size by ~33% and cost real time on the
+	// WiFi-constrained upload path). Read it straight off the request
+	// body.
+	decoded, err := io.ReadAll(r.Body)
 	if err != nil {
-		log.Fatal(err)
-	}
-	if _, err := f.Write(decoded); err != nil {
-		log.Fatal(err)
+		http.Error(w, "Failed to read chunk body", http.StatusBadRequest)
+		return
 	}
-	if err := f.Close(); err != nil {
-		log.Fatal(err)
+
+	// state.File is opened once in uploadStart and closed in
+	// uploadFinish/uploadCancel - opening, writing and closing it on
+	// every chunk here was real per-chunk overhead (syscalls plus an
+	// implicit flush on close), especially costly against the
+	// FAT-formatted USB target.
+	//
+	// Chunks are sent one at a time (not pipelined/concurrent) - this
+	// board's USB hub (WiFi NIC + storage share a single Transaction
+	// Translator, confirmed from the hub's datasheet) can't reliably
+	// handle simultaneous network-receive and disk-write transactions.
+	// Concurrent chunks caused a real disk-write pileup (kernel threads
+	// stuck in D-state) and, even after serializing the writes,
+	// intermittent connection resets - a hardware constraint, not
+	// something fixable by tuning the write path further.
+	if _, err := state.File.Write(decoded); err != nil {
+		http.Error(w, "Failed to write chunk to file", http.StatusInternalServerError)
+		return
 	}
 	state.BytesNow += len(decoded)
 	state.Progress = float64(state.BytesNow) * 100 / float64(state.BytesTotal)
@@ -702,6 +716,12 @@ func uploadChunk(w http.ResponseWriter, r *http.Request) {
 }
 
 func uploadFinish(w http.ResponseWriter, r *http.Request) {
+	if state.File != nil {
+		if err := state.File.Close(); err != nil {
+			log.Fatal(err)
+		}
+		state.File = nil
+	}
 	mountUsb(MODE_RO)
 	duration := time.Since(timeStart)
 	logInfo(fmt.Sprintf("Upload finished in %d minutes and %d seconds", int(duration.Minutes()), int(duration.Seconds())%60))

reflash/server_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"net/http/httptest"
 	"os"
@@ -393,3 +394,63 @@ func TestGetUncompressedSizeRealXz(t *testing.T) {
 		t.Errorf("getUncompressedSize = %d, want %d", got, size)
 	}
 }
+
+// End-to-end against the real handlers: drives upload_start -> several
+// upload_chunk calls -> upload_finish exactly like the client does, and
+// checks the file on disk matches byte-for-byte. Exercises the real
+// state.File handle kept open across chunks (rather than the old
+// open/write/close-per-chunk code), so this would catch truncation,
+// overwriting, or interleaving bugs a build-only check can't.
+func TestUploadChunkRoundTrip(t *testing.T) {
+	setupTest(t)
+	state = &State{State: IDLE}
+
+	filename := "roundtrip-test.img.xz"
+	payload := bytes.Repeat([]byte("The quick brown fox jumps over the lazy dog. "), 5000) // ~225KB
+
+	startBody, _ := json.Marshal(map[string]any{
+		"filename":   filename,
+		"size":       len(payload),
+		"start_time": 0,
+	})
+	w := httptest.NewRecorder()
+	uploadStart(w, httptest.NewRequest("PUT", "/api/upload_start", bytes.NewReader(startBody)))
+	if state.File == nil {
+		t.Fatal("uploadStart did not open state.File")
+	}
+
+	const chunkSize = 10 * 1024 // small relative to payload so this genuinely exercises multiple sequential chunk writes, not just one
+	for i := 0; i < len(payload); i += chunkSize {
+		end := i + chunkSize
+		if end > len(payload) {
+			end = len(payload)
+		}
+		w := httptest.NewRecorder()
+		uploadChunk(w, httptest.NewRequest("POST", "/api/upload_chunk", bytes.NewReader(payload[i:end])))
+
+		var resp map[string]bool
+		if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
+			t.Fatalf("chunk at offset %d: bad response body: %v", i, err)
+		}
+		if !resp["success"] {
+			t.Fatalf("chunk at offset %d: server reported failure", i)
+		}
+	}
+
+	uploadFinish(httptest.NewRecorder(), httptest.NewRequest("PUT", "/api/upload_finish", nil))
+
+	if state.File != nil {
+		t.Error("uploadFinish did not clear state.File")
+	}
+
+	got, err := os.ReadFile(filepath.Join(images_folder, filename))
+	if err != nil {
+		t.Fatalf("reading uploaded file: %v", err)
+	}
+	if !bytes.Equal(got, payload) {
+		t.Errorf("uploaded file mismatch: got %d bytes, want %d bytes", len(got), len(payload))
+	}
+	if state.BytesNow != len(payload) {
+		t.Errorf("state.BytesNow = %d, want %d", state.BytesNow, len(payload))
+	}
+}