james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

prevent running out of file descriptors

Browse Source
This commit is contained in:
Nazar Kanaev 2020-09-10 11:35:31 +01:00
parent c547f552dd
commit 0dac50a84d
1 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
server/crawler.go
View File

@ -138,14 +138,20 @@ func findFavicon(websiteUrl, feedUrl string) (*[]byte, error) {
}
if len(websiteUrl) != 0 {
doc, err := goquery.NewDocument(websiteUrl)
res, err := defaultClient.get(websiteUrl)
if err != nil {
return nil, err
}
defer res.Body.Close()
doc, err := goquery.NewDocumentFromReader(res.Body)
base, err := url.Parse(websiteUrl)
if err != nil {
return nil, err
}
doc.Find(`link[rel=icon]`).EachWithBreak(func(i int, s *goquery.Selection) bool {
if href, ok := s.Attr("href"); ok {
if hrefUrl, err := url.Parse(href); err == nil {
faviconUrl := doc.Url.ResolveReference(hrefUrl).String()
faviconUrl := base.ResolveReference(hrefUrl).String()
candidateUrls = append(candidateUrls, faviconUrl)
}
}
@ -233,8 +239,14 @@ func listItems(f storage.Feed) ([]storage.Item, error) {
}
func init() {
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.DisableKeepAlives = true
httpClient := &http.Client{
Timeout: time.Second * 5,
Transport: transport,
}
defaultClient = &Client{
httpClient: &http.Client{Timeout: time.Second * 5},
httpClient: httpClient,
userAgent: "Yarr/1.0",
}
}