From 95f15c5808ab31c9f6243401161b94c2f78266c2 Mon Sep 17 00:00:00 2001 From: Himadri Bhattacharjee Date: Fri, 14 Jul 2023 07:06:36 +0530 Subject: [PATCH] refactor: return client structure when API URL is acquired --- pkg/providers/commoncrawl/commoncrawl.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/pkg/providers/commoncrawl/commoncrawl.go b/pkg/providers/commoncrawl/commoncrawl.go index 920a71b..bac06a7 100644 --- a/pkg/providers/commoncrawl/commoncrawl.go +++ b/pkg/providers/commoncrawl/commoncrawl.go @@ -29,7 +29,6 @@ type Client struct { } func New(c *providers.Config, filters providers.Filters) (*Client, error) { - client := &Client{config: c, filters: filters} // Fetch the list of available CommonCrawl Api URLs. resp, err := httpclient.MakeRequest(c.Client, "http://index.commoncrawl.org/collinfo.json", c.MaxRetries, c.Timeout) if err != nil { @@ -45,8 +44,7 @@ func New(c *providers.Config, filters providers.Filters) (*Client, error) { return nil, errors.New("failed to grab latest commoncrawl index") } - client.apiURL = r[0].API - return client, nil + return &Client{config: c, filters: filters, apiURL: r[0].API}, nil } func (c *Client) Name() string { @@ -111,18 +109,15 @@ func (c *Client) formatURL(domain string, page uint) string { } // Fetch the number of pages. -func (c *Client) getPagination(domain string) (paginationResult, error) { +func (c *Client) getPagination(domain string) (r paginationResult, err error) { url := fmt.Sprintf("%s&showNumPages=true", c.formatURL(domain, 0)) + var resp []byte - resp, err := httpclient.MakeRequest(c.config.Client, url, c.config.MaxRetries, c.config.Timeout) + resp, err = httpclient.MakeRequest(c.config.Client, url, c.config.MaxRetries, c.config.Timeout) if err != nil { - return paginationResult{}, err + return } - var r paginationResult - if err = jsoniter.Unmarshal(resp, &r); err != nil { - return r, err - } - - return r, nil + err = jsoniter.Unmarshal(resp, &r) + return }