-
Notifications
You must be signed in to change notification settings - Fork 34
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(scraper): add external Web Scraping service
- Loading branch information
Showing
13 changed files
with
215 additions
and
96 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package scraper | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "net/http" | ||
| "net/url" | ||
| "time" | ||
|
|
||
| "github.com/rs/zerolog" | ||
| "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| type extrenalWebScraper struct { | ||
| uri string | ||
| logger zerolog.Logger | ||
| } | ||
|
|
||
| // NewExternalWebScraper create an external web scrapping service | ||
| func NewExternalWebScraper(uri string) (WebScraper, error) { | ||
| if _, err := url.ParseRequestURI(uri); err != nil { | ||
| return nil, fmt.Errorf("invalid Web Scraping service URI: %s", uri) | ||
| } | ||
| logger := log.With().Str("component", "webscraper").Str("uri", uri).Logger() | ||
| logger.Debug().Msg("using external service") | ||
|
|
||
| return &extrenalWebScraper{ | ||
| uri: uri, | ||
| logger: logger, | ||
| }, nil | ||
| } | ||
|
|
||
| func (ws extrenalWebScraper) Scrap(ctx context.Context, url string) (*WebPage, error) { | ||
| webPage, err := ws.scrap(ctx, url) | ||
| if err != nil { | ||
| ws.logger.Error().Err(err).Msg("unable to scrap web page with external service, fallback on internal service") | ||
| return NewInternalWebScraper().Scrap(ctx, url) | ||
| } | ||
| return webPage, nil | ||
| } | ||
|
|
||
| func (ws extrenalWebScraper) scrap(ctx context.Context, url string) (*WebPage, error) { | ||
| ctx, cancel := context.WithTimeout(ctx, 5*time.Second) | ||
| defer cancel() | ||
| req, err := http.NewRequestWithContext(ctx, "GET", ws.uri, nil) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| q := req.URL.Query() | ||
| q.Add("u", url) | ||
| req.URL.RawQuery = q.Encode() | ||
|
|
||
| ws.logger.Debug().Str("url", url).Msg("scraping webpage") | ||
| res, err := http.DefaultClient.Do(req) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer res.Body.Close() | ||
|
|
||
| if res.StatusCode >= 400 { | ||
| return nil, fmt.Errorf("invalid web scraping response: %d", res.StatusCode) | ||
| } | ||
|
|
||
| if ct := res.Header.Get("Content-Type"); ct != "" { | ||
| if ct != "application/json" { | ||
| return nil, fmt.Errorf("invalid web scraping Content-Type response: %s", ct) | ||
| } | ||
| } | ||
|
|
||
| webPage := WebPage{} | ||
| err = json.NewDecoder(res.Body).Decode(&webPage) | ||
| return &webPage, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package scraper | ||
|
|
||
| import ( | ||
| "context" | ||
| ) | ||
|
|
||
| // WebPage is the result of a web scraping | ||
| type WebPage struct { | ||
| Title string `json:"title,omitempty"` | ||
| HTML string `json:"html,omitempty"` | ||
| Text string `json:"text,omitempty"` | ||
| Length int `json:"length,omitempty"` | ||
| Excerpt string `json:"excerpt,omitempty"` | ||
| SiteName string `json:"sitename,omitempty"` | ||
| Image string `json:"image,omitempty"` | ||
| Favicon string `json:"favicon,omitempty"` | ||
| } | ||
|
|
||
| // WebScraper is an interface with Web Scrapping provider | ||
| type WebScraper interface { | ||
| Scrap(ctx context.Context, url string) (*WebPage, error) | ||
| } | ||
|
|
||
| // NewWebScraper create new Web Scraping service | ||
| func NewWebScraper(uri string) (WebScraper, error) { | ||
| if uri == "" { | ||
| return NewInternalWebScraper(), nil | ||
| } | ||
| return NewExternalWebScraper(uri) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.