diff --git a/tests/http-range-file/README.md b/tests/http-range-file/README.md new file mode 100644 index 000000000..4b6ad3421 --- /dev/null +++ b/tests/http-range-file/README.md @@ -0,0 +1,7 @@ +# Description + +Test http file extraction over multiple transactions with range header. + +# PCAP + +The pcap comes from running `go run client.go` diff --git a/tests/http-range-file/client.go b/tests/http-range-file/client.go new file mode 100644 index 000000000..61a2df315 --- /dev/null +++ b/tests/http-range-file/client.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strconv" + "strings" + "time" +) + +func main() { + url := "http://i.imgur.com/z4d4kWk.jpg" + step := 10000 + + tr := &http.Transport{ + //may not be needed + MaxIdleConns: 10, + MaxIdleConnsPerHost: 10, + MaxConnsPerHost: 10, + IdleConnTimeout: 30 * time.Second, + DisableKeepAlives: false, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + } + client := &http.Client{Transport: tr} + + req, _ := http.NewRequest("GET", url, nil) + req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", step-1)) + resp, err := client.Do(req) + filesize, _ := strconv.Atoi(strings.Split(resp.Header["Content-Range"][0], "/")[1]) + fmt.Printf("%#+v %#+v\n", filesize, err) + //cf https://github.com/golang/go/issues/26095 + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + + for start := step; start < filesize; start += step { + req2, _ := http.NewRequest("GET", url, nil) + req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, start+step-1)) + resp2, _ := client.Do(req2) + io.Copy(ioutil.Discard, resp2.Body) + resp2.Body.Close() + + fmt.Printf("%#+v %#+v\n", start, step) + } +} diff --git a/tests/http-range-file/input.pcap b/tests/http-range-file/input.pcap new file mode 100644 index 000000000..4cae52d43 Binary files /dev/null and b/tests/http-range-file/input.pcap differ diff --git a/tests/http-range-file/test.yaml b/tests/http-range-file/test.yaml new file mode 100644 index 000000000..fcb5b4dde --- /dev/null +++ b/tests/http-range-file/test.yaml @@ -0,0 +1,17 @@ +requires: + features: + - HAVE_LIBJANSSON + min-version: 7.0.0 + +# disables checksum verification +args: + - -k none + +checks: + + # Check that there is one file event with content range. + - filter: + count: 1 + match: + event_type: fileinfo + fileinfo.size: 146515 diff --git a/tests/http-range-multiflows/README.md b/tests/http-range-multiflows/README.md new file mode 100644 index 000000000..1d0113ff5 --- /dev/null +++ b/tests/http-range-multiflows/README.md @@ -0,0 +1,10 @@ +# Description + +Test http file extraction over multiple flows with range header (unordered). + +# PCAP + +The pcap comes from running `go run client.go` +The server running in the background is `python3 -m RangeHTTPServer` +in directory mqtt-binary-message using https://github.com/danvk/RangeHTTPServer + diff --git a/tests/http-range-multiflows/client.go b/tests/http-range-multiflows/client.go new file mode 100644 index 000000000..e2d01c8bc --- /dev/null +++ b/tests/http-range-multiflows/client.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strconv" + "strings" + "time" +) + +type httpRange struct { + Start uint + End uint +} + +const step = 1000 +const url = "http://127.0.0.1:8000/mqtt5_pub_jpeg.pcap" + +func main() { + + tr := &http.Transport{ + //may not be needed + MaxIdleConns: 10, + MaxIdleConnsPerHost: 10, + MaxConnsPerHost: 10, + IdleConnTimeout: 30 * time.Second, + DisableKeepAlives: false, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + } + client := &http.Client{Transport: tr} + myranges := []httpRange{ + {1000, 2000}, // out of order + {0, 1000}, // order + resume previous + {500, 1500}, // only overlap + {1500, 2500}, // overlap + new data + {5000, 6000}, // out of order + {2500, 3500}, // order but no resume + {4000, 5000}, // out of order insert in head + {8000, 9000}, // out or order insert at tail + {6000, 7000}, // out of order insert in the middle + {6400, 8000}, // insert with overlap + {3000, 4000}, // overlap + new data + resume multiple + } + + filesize := 0 + + for i := range myranges { + req2, _ := http.NewRequest("GET", url, nil) + req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", myranges[i].Start, myranges[i].End-1)) + resp2, _ := client.Do(req2) + filesize, _ = strconv.Atoi(strings.Split(resp2.Header["Content-Range"][0], "/")[1]) + io.Copy(ioutil.Discard, resp2.Body) + resp2.Body.Close() + + fmt.Printf("download %#+v %#+v\n", myranges[i].Start, step) + } + + for o := 8000; o < filesize; o += step { + req2, _ := http.NewRequest("GET", url, nil) + req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", o, o+step-1)) + resp2, _ := client.Do(req2) + io.Copy(ioutil.Discard, resp2.Body) + resp2.Body.Close() + fmt.Printf("download %#+v %#+v\n", o, step) + } +} diff --git a/tests/http-range-multiflows/input.pcap b/tests/http-range-multiflows/input.pcap new file mode 100644 index 000000000..438ae8ae1 Binary files /dev/null and b/tests/http-range-multiflows/input.pcap differ diff --git a/tests/http-range-multiflows/test.yaml b/tests/http-range-multiflows/test.yaml new file mode 100644 index 000000000..9686cd92f --- /dev/null +++ b/tests/http-range-multiflows/test.yaml @@ -0,0 +1,21 @@ +requires: + features: + - HAVE_LIBJANSSON + min-version: 7.0.0 + +# disables checksum verification +args: + - -k none +# we want to check every packet in pcap order + - --set runmode=single +# make one alloc fail and the test fail if we limit memcap +# - --set containers.urlrange.memcap=195000 + +checks: + + # Check that there is one file event with content range. + - filter: + count: 1 + match: + event_type: fileinfo + fileinfo.size: 37323 diff --git a/tests/http2-range/README.md b/tests/http2-range/README.md new file mode 100644 index 000000000..3ea272eb8 --- /dev/null +++ b/tests/http2-range/README.md @@ -0,0 +1,14 @@ +# Description + +Test http2 file extraction over multiple transactions with range header. +# PCAP + +The pcap comes from running dummy HTTP1 and HTTP2 server with `go run server.go` with an eicar.txt file in the current directory containing the eicar file +and in parallel as client(s) : +``` +curl -H 'Range: bytes=0-10' --http2 127.0.0.1:8080/eicar +curl -H 'Range: bytes=10-20' 127.0.0.1:8080/eicar +curl -H 'Range: bytes=20-30' --http2 127.0.0.1:8080/eicar +curl -H 'Range: bytes=30-40' 127.0.0.1:8080/eicar +curl -H 'Range: bytes=40-68' --http2 127.0.0.1:8080/eicar +``` diff --git a/tests/http2-range/http2-range.pcap b/tests/http2-range/http2-range.pcap new file mode 100644 index 000000000..0f149a6ca Binary files /dev/null and b/tests/http2-range/http2-range.pcap differ diff --git a/tests/http2-range/server.go b/tests/http2-range/server.go new file mode 100644 index 000000000..1abcb6ad2 --- /dev/null +++ b/tests/http2-range/server.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" + "net/http" +) + +func main() { + h2s := &http2.Server{} + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, "eicar.txt") + }) + + server := &http.Server{ + Addr: "0.0.0.0:8080", + Handler: h2c.NewHandler(handler, h2s), + } + + fmt.Printf("Listening [0.0.0.0:8080]...\n") + err := server.ListenAndServe() + fmt.Printf("lol %s", err) +} diff --git a/tests/http2-range/test.yaml b/tests/http2-range/test.yaml new file mode 100644 index 000000000..a3b204d97 --- /dev/null +++ b/tests/http2-range/test.yaml @@ -0,0 +1,21 @@ +requires: + features: + - HAVE_LIBJANSSON + min-version: 7.0.0 + +# disables checksum verification +args: + - -k none --set app-layer.protocols.http2.enabled=true + +checks: + + # Check that there is one file event with total length. + - filter: + count: 1 + match: + event_type: fileinfo + fileinfo.size: 69 + - filter: + count: 0 + match: + event_type: anomaly