Http range file v4

tests/http-range-file/README.md

@@ -0,0 +1,7 @@
+# Description
+
+Test http file extraction over multiple transactions with range header.
+
+# PCAP
+
+The pcap comes from running `go run client.go`

tests/http-range-file/client.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func main() {
+	url := "http://i.imgur.com/z4d4kWk.jpg"
+	step := 10000
+
+	tr := &http.Transport{
+		//may not be needed
+		MaxIdleConns:        10,
+		MaxIdleConnsPerHost: 10,
+		MaxConnsPerHost:     10,
+		IdleConnTimeout:     30 * time.Second,
+		DisableKeepAlives:   false,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+	}
+	client := &http.Client{Transport: tr}
+
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", step-1))
+	resp, err := client.Do(req)
+	filesize, _ := strconv.Atoi(strings.Split(resp.Header["Content-Range"][0], "/")[1])
+	fmt.Printf("%#+v %#+v\n", filesize, err)
+	//cf https://github.com/golang/go/issues/26095
+	io.Copy(ioutil.Discard, resp.Body)
+	resp.Body.Close()
+
+	for start := step; start < filesize; start += step {
+		req2, _ := http.NewRequest("GET", url, nil)
+		req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, start+step-1))
+		resp2, _ := client.Do(req2)
+		io.Copy(ioutil.Discard, resp2.Body)
+		resp2.Body.Close()
+
+		fmt.Printf("%#+v %#+v\n", start, step)
+	}
+}

tests/http-range-file/test.yaml

@@ -0,0 +1,17 @@
+requires:
+  features:
+    - HAVE_LIBJANSSON
+  min-version: 7.0.0
+
+# disables checksum verification
+args:
+  - -k none
+
+checks:
+
+  # Check that there is one file event with content range.
+  - filter:
+      count: 1
+      match:
+        event_type: fileinfo
+        fileinfo.size: 146515

tests/http-range-multiflows/README.md

@@ -0,0 +1,10 @@
+# Description
+
+Test http file extraction over multiple flows with range header (unordered).
+
+# PCAP
+
+The pcap comes from running `go run client.go`
+The server running in the background is `python3 -m RangeHTTPServer`
+in directory mqtt-binary-message using https://github.com/danvk/RangeHTTPServer
+

tests/http-range-multiflows/client.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type httpRange struct {
+	Start uint
+	End   uint
+}
+
+const step = 1000
+const url = "http://127.0.0.1:8000/mqtt5_pub_jpeg.pcap"
+
+func main() {
+
+	tr := &http.Transport{
+		//may not be needed
+		MaxIdleConns:        10,
+		MaxIdleConnsPerHost: 10,
+		MaxConnsPerHost:     10,
+		IdleConnTimeout:     30 * time.Second,
+		DisableKeepAlives:   false,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+	}
+	client := &http.Client{Transport: tr}
+	myranges := []httpRange{
+		{1000, 2000}, // out of order
+		{0, 1000},    // order + resume previous
+		{500, 1500},  // only overlap
+		{1500, 2500}, // overlap + new data
+		{5000, 6000}, // out of order
+		{2500, 3500}, // order but no resume
+		{4000, 5000}, // out of order insert in head
+		{8000, 9000}, // out or order insert at tail
+		{6000, 7000}, // out of order insert in the middle
+		{6400, 8000}, // insert with overlap
+		{3000, 4000}, // overlap + new data + resume multiple
+	}
+
+	filesize := 0
+
+	for i := range myranges {
+		req2, _ := http.NewRequest("GET", url, nil)
+		req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", myranges[i].Start, myranges[i].End-1))
+		resp2, _ := client.Do(req2)
+		filesize, _ = strconv.Atoi(strings.Split(resp2.Header["Content-Range"][0], "/")[1])
+		io.Copy(ioutil.Discard, resp2.Body)
+		resp2.Body.Close()
+
+		fmt.Printf("download %#+v %#+v\n", myranges[i].Start, step)
+	}
+
+	for o := 8000; o < filesize; o += step {
+		req2, _ := http.NewRequest("GET", url, nil)
+		req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", o, o+step-1))
+		resp2, _ := client.Do(req2)
+		io.Copy(ioutil.Discard, resp2.Body)
+		resp2.Body.Close()
+		fmt.Printf("download %#+v %#+v\n", o, step)
+	}
+}

tests/http-range-multiflows/test.yaml

@@ -0,0 +1,21 @@
+requires:
+  features:
+    - HAVE_LIBJANSSON
+  min-version: 7.0.0
+
+# disables checksum verification
+args:
+  - -k none
+# we want to check every packet in pcap order
+  - --set runmode=single
+# make one alloc fail and the test fail if we limit memcap
+#  - --set containers.urlrange.memcap=195000
+
+checks:
+
+  # Check that there is one file event with content range.
+  - filter:
+      count: 1
+      match:
+        event_type: fileinfo
+        fileinfo.size: 37323

tests/http2-range/README.md

@@ -0,0 +1,14 @@
+# Description
+
+Test http2 file extraction over multiple transactions with range header.
+# PCAP
+
+The pcap comes from running dummy HTTP1 and HTTP2 server with `go run server.go` with an eicar.txt file in the current directory containing the eicar file
+and in parallel as client(s) :
+```
+curl -H 'Range: bytes=0-10' --http2 127.0.0.1:8080/eicar
+curl -H 'Range: bytes=10-20' 127.0.0.1:8080/eicar
+curl -H 'Range: bytes=20-30' --http2 127.0.0.1:8080/eicar
+curl -H 'Range: bytes=30-40' 127.0.0.1:8080/eicar
+curl -H 'Range: bytes=40-68' --http2 127.0.0.1:8080/eicar
+```

tests/http2-range/server.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+	"net/http"
+)
+
+func main() {
+	h2s := &http2.Server{}
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "eicar.txt")
+	})
+
+	server := &http.Server{
+		Addr:    "0.0.0.0:8080",
+		Handler: h2c.NewHandler(handler, h2s),
+	}
+
+	fmt.Printf("Listening [0.0.0.0:8080]...\n")
+	err := server.ListenAndServe()
+	fmt.Printf("lol %s", err)
+}

tests/http2-range/test.yaml

@@ -0,0 +1,21 @@
+requires:
+  features:
+    - HAVE_LIBJANSSON
+  min-version: 7.0.0
+
+# disables checksum verification
+args:
+  - -k none --set app-layer.protocols.http2.enabled=true
+
+checks:
+
+  # Check that there is one file event with total length.
+  - filter:
+      count: 1
+      match:
+        event_type: fileinfo
+        fileinfo.size: 69
+  - filter:
+      count: 0
+      match:
+        event_type: anomaly