diff --git a/README.md b/README.md index 765fc1a..cb347fb 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,12 @@ Flags: Uploads larger than 1 MB are spooled to temporary files under `$TMPDIR` (default `/tmp`) rather than held in memory. The temporary files are removed after each request. +If `/tmp` is a RAM-backed tmpfs, as on Raspberry Pi OS, uploads still use RAM, and a file bigger than the tmpfs fails. In that case point `TMPDIR` at a disk directory: + +```sh +TMPDIR=/path/on/disk ./pcap-web-tools -max-upload-mb 5000 +``` + ## Tools ### Hex → pcap (`/hex2pcap`) @@ -54,12 +60,35 @@ Splits an uploaded pcap file into parts with at most N packets or at most N KB/M - The upload is checked completely before the zip is sent, so errors show on the page instead of as a broken download. - Only classic pcap is supported. Convert pcapng first with `editcap -F pcap in.pcapng out.pcap`. +### Reorder pcap (`/reorder`) + +Sorts the packets of an uploaded pcap file by timestamp and returns the result as `_reordered.pcap`. The output is byte-identical to `reordercap`'s. + +- The sort is stable, so packets with equal timestamps keep their original order. +- The file header and packet records are copied unchanged. +- The response headers report the number of packets (`X-Pcap-Packets`), the number of packets that were out of order (`X-Pcap-Out-Of-Order`) and whether a truncated final packet was dropped (`X-Pcap-Truncated`). These are useful when scripting with `curl -F file=@in.pcap`. + +How it works: + +- **Index:** one sequential pass keeps a 16-byte index entry per packet (timestamp, plus offset and length), which is then sorted. +- **Write:** the packets are copied out in sorted order using 16 parallel reads in 8 MB batches, reading the next batch while the previous one is written. SSDs are much faster at random reads with many requests in flight. +- **Already sorted:** if the file is already in order, it is copied straight through. + +Measured on a Raspberry Pi 5 with NVMe, for 3 million 1500-byte packets (4.5 GB) with fully random timestamps, the worst case: + +| | Time | Peak memory | +|---|---|---| +| Server: upload, reorder, download over localhost | 95 s | 108 MB | +| `reordercap` | about 3.5 h (extrapolated from 0.35 MB/s) | 180 MB | + +Memory is roughly 16 bytes per packet for the index, plus about 16 MB of copy buffers and Go GC headroom. Captures that are mostly in order, the usual case, read nearly sequentially and are faster. + ## Layout ``` main.go entry point, HTTP server internal/web/ handlers, tool registry, embedded templates and CSS -internal/pcap/ pcap file reader, writer and splitter +internal/pcap/ pcap file reader, writer, splitter and reorderer internal/inet/ IPv4/IPv6 length and checksum fixups, IP header locator internal/hexutil/ lenient hex decoding ``` diff --git a/internal/web/server.go b/internal/web/server.go index cf83a21..4e42897 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -23,6 +23,7 @@ type Tool struct { var tools = []Tool{ {Name: "Hex → pcap", Path: "/hex2pcap", Description: "Build a pcap file from hex-encoded layer 2, layer 3 and payload bytes, optionally fixing IP lengths and checksums."}, {Name: "Split pcap", Path: "/split", Description: "Split a pcap file into several files by packet count or file size, downloaded as a zip."}, + {Name: "Reorder pcap", Path: "/reorder", Description: "Sort the packets of a pcap file by timestamp, like reordercap."}, } // Config holds server settings. @@ -43,7 +44,7 @@ func New(cfg Config) (*Server, error) { // Each page is parsed together with the shared layout so pages can define // their own "title" and "content" blocks. - for _, page := range []string{"index.html", "hex2pcap.html", "split.html"} { + for _, page := range []string{"index.html", "hex2pcap.html", "split.html", "reorder.html"} { t, err := template.New(page).ParseFS(assets, "templates/layout.html", "templates/"+page) if err != nil { return nil, err @@ -61,6 +62,8 @@ func New(cfg Config) (*Server, error) { s.mux.HandleFunc("POST /hex2pcap", s.handleHex2Pcap) s.mux.HandleFunc("GET /split", s.handleSplitForm) s.mux.HandleFunc("POST /split", s.handleSplit) + s.mux.HandleFunc("GET /reorder", s.handleReorderForm) + s.mux.HandleFunc("POST /reorder", s.handleReorder) return s, nil } diff --git a/internal/web/split.go b/internal/web/split.go index e2d787c..b62c7e6 100644 --- a/internal/web/split.go +++ b/internal/web/split.go @@ -9,7 +9,6 @@ import ( "log" "math" "net/http" - "path" "strconv" "strings" "time" @@ -50,7 +49,7 @@ func (s *Server) splitPage(f splitForm) splitPage { return splitPage{ Form: f, Units: units, - MaxUpload: fmt.Sprintf("%d MB", s.cfg.MaxUploadBytes/1e6), + MaxUpload: s.maxUpload(), MaxParts: maxSplitParts, } } @@ -67,19 +66,12 @@ func (s *Server) handleSplit(w http.ResponseWriter, r *http.Request) { s.render(w, status, "split.html", "/split", page) } - // Allow some slack over the file size limit for the multipart framing and form fields. - r.Body = http.MaxBytesReader(w, r.Body, s.cfg.MaxUploadBytes+64<<10) - // Uploads bigger than 1 MB are spooled to temporary files rather than kept in memory. - if err := r.ParseMultipartForm(1 << 20); err != nil { - var tooBig *http.MaxBytesError - if errors.As(err, &tooBig) { - fail(http.StatusRequestEntityTooLarge, "file too large; the maximum upload is "+page.MaxUpload) - } else { - fail(http.StatusBadRequest, "bad upload: "+err.Error()) - } + file, hdr, status, msg := s.parseUpload(w, r) + if status != 0 { + fail(status, msg) return } - defer r.MultipartForm.RemoveAll() + defer file.Close() page.Form = splitForm{ Mode: r.FormValue("mode"), @@ -92,13 +84,6 @@ func (s *Server) handleSplit(w http.ResponseWriter, r *http.Request) { fail(http.StatusBadRequest, err.Error()) return } - file, hdr, err := r.FormFile("file") - if err != nil { - fail(http.StatusBadRequest, "choose a pcap file to split") - return - } - defer file.Close() - // First pass: read the whole file and count the parts, so problems are // reported on the page before any of the zip is sent. st, err := pcap.Split(file, limit, func(part int) (io.Writer, error) { @@ -116,8 +101,7 @@ func (s *Server) handleSplit(w http.ResponseWriter, r *http.Request) { return } - name := safeName(hdr.Filename, "capture.pcap") - base := strings.TrimSuffix(name, path.Ext(name)) + base := uploadStem(hdr) digits := max(3, len(strconv.Itoa(st.Parts))) w.Header().Set("Content-Type", "application/zip") diff --git a/internal/web/split_test.go b/internal/web/split_test.go index c85cf5c..13942c4 100644 --- a/internal/web/split_test.go +++ b/internal/web/split_test.go @@ -28,7 +28,7 @@ func testPcap(t *testing.T, packets int) []byte { return buf.Bytes() } -func postSplit(t *testing.T, srv *Server, filename string, file []byte, fields map[string]string) *httptest.ResponseRecorder { +func postUpload(t *testing.T, srv *Server, target, filename string, file []byte, fields map[string]string) *httptest.ResponseRecorder { t.Helper() var body bytes.Buffer mw := multipart.NewWriter(&body) @@ -43,7 +43,7 @@ func postSplit(t *testing.T, srv *Server, filename string, file []byte, fields m fw.Write(file) } mw.Close() - req := httptest.NewRequest("POST", "/split", &body) + req := httptest.NewRequest("POST", target, &body) req.Header.Set("Content-Type", mw.FormDataContentType()) rec := httptest.NewRecorder() srv.ServeHTTP(rec, req) @@ -68,7 +68,7 @@ func TestSplitHandler(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rec := postSplit(t, srv, `C:\caps\my capture.pcap`, in, tt.fields) + rec := postUpload(t, srv, "/split", `C:\caps\my capture.pcap`, in, tt.fields) if rec.Code != http.StatusOK || rec.Header().Get("Content-Type") != "application/zip" { t.Fatalf("status %d %s: %s", rec.Code, rec.Header().Get("Content-Type"), rec.Body) } @@ -120,7 +120,7 @@ func TestSplitHandlerErrors(t *testing.T) { {"no mode", testPcap(t, 1), map[string]string{}, http.StatusBadRequest, "choose whether"}, } for _, tt := range tests { - rec := postSplit(t, srv, "x.pcap", tt.file, tt.fields) + rec := postUpload(t, srv, "/split", "x.pcap", tt.file, tt.fields) if rec.Code != tt.status || !strings.Contains(rec.Body.String(), tt.want) { t.Errorf("%s: status %d, body %.300s", tt.name, rec.Code, rec.Body) } diff --git a/main.go b/main.go index 8ee3f01..7d9ae75 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "flag" "log" "net/http" + "os" "time" "pcap-web-tools/internal/web" @@ -25,5 +26,6 @@ func main() { ReadHeaderTimeout: 10 * time.Second, } log.Printf("listening on http://%s", *addr) + log.Printf("uploads larger than 1 MB are spooled to %s (set TMPDIR to change)", os.TempDir()) log.Fatal(httpSrv.ListenAndServe()) }