From 395b542b56380d8bb52d049b4df9b9e426adec9d Mon Sep 17 00:00:00 2001 From: Hazana Date: Wed, 28 Dec 2022 14:43:23 +0000 Subject: [PATCH 1/3] Add random useragent --- second-order.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/second-order.go b/second-order.go index 19ae077..f120f1c 100644 --- a/second-order.go +++ b/second-order.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "log" + "math/rand" "net/http" "net/url" "os" @@ -125,6 +126,11 @@ func main() { // Add headers c.OnRequest(func(r *colly.Request) { + // Set a random user agent for each request + rand.Seed(time.Now().Unix()) + n := rand.Intn(len(userAgents)) + r.Headers.Set("User-Agent", userAgents[n]) + // Add other headers for header, value := range headers { r.Headers.Set(header, value) } @@ -336,3 +342,30 @@ func isNotFound(url string) bool { } return false } + +var userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.54", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.2 Safari/605.1.15", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36 Edg/108.0.1462.46", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 Edg/92.0.902.67", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0; rv:87.0) Gecko/20100101 Firefox/87.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/87.0.664.59", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36 Edg/87.0.664.56", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_0_0; rv:85.0) Gecko/20100101 Firefox/85.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:85.0) Gecko/20100101 Firefox/85.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.72", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36", +} From 176e483f82a2862a73c49ec4f0813f2c5ac455fd Mon Sep 17 00:00:00 2001 From: Hazana Date: Wed, 28 Dec 2022 16:43:37 +0000 Subject: [PATCH 2/3] Add results type to output --- second-order.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/second-order.go b/second-order.go index f120f1c..ad34dc2 100644 --- a/second-order.go +++ b/second-order.go @@ -215,19 +215,19 @@ func writeAllResults(config Configuration) { os.MkdirAll(outdir, os.ModePerm) if config.LogQueries != nil { - err := writeResults("attributes.json", loggedQueries.content) + err := writeResults("attributes.json", loggedQueries.content, "LogQueries") if err != nil { log.Printf("Error writing attributes: %v", err) } } if config.LogInline != nil { - err := writeResults("inline.json", loggedInline.content) + err := writeResults("inline.json", loggedInline.content, "LogInline") if err != nil { log.Printf("Error writing inline text: %v", err) } } if config.LogNon200Queries != nil { - err := writeResults("non-200-url-attributes.json", loggedNon200Queries.content) + err := writeResults("non-200-url-attributes.json", loggedNon200Queries.content, "LogNon200Queries") if err != nil { log.Printf("Error writing non-200 URL attributes: %v", err) } @@ -264,8 +264,10 @@ func unpackQuerySelector(q string) (string, string) { return tag, attribute } -func writeResults(filename string, content map[string]map[string][]string) error { - JSON, err := json.Marshal(content) +func writeResults(filename string, content map[string]map[string][]string, resultType string) error { + output := make(map[string]map[string]map[string][]string) + output[resultType] = content + JSON, err := json.Marshal(output) if err != nil { return fmt.Errorf("could not marshal the JSON object: %v", err) } From d091b4bc04c31594bc002fded2da48a910606429 Mon Sep 17 00:00:00 2001 From: Hazana Date: Wed, 28 Dec 2022 16:53:15 +0000 Subject: [PATCH 3/3] Create go.yml --- .github/workflows/go.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..b545940 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,15 @@ +on: + release: + types: [created] + +jobs: + release-linux-amd64: + name: release linux/amd64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: wangyoucao577/go-release-action@v1.34 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + goos: linux + goarch: amd64