james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 21:19:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

conditional http get

Browse Source
This commit is contained in:
Nazar Kanaev 2020-10-17 13:27:12 +01:00
parent 11a2aa2b4a
commit f27d0c4cd7
2 changed files with 35 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
server/crawler.go
View File

@ -47,6 +47,17 @@ func (c *Client) get(url string) (*http.Response, error) {
return c.httpClient.Do(req) return c.httpClient.Do(req)
} }
func (c *Client) getConditional(url, lastModified, etag string) (*http.Response, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", c.userAgent)
req.Header.Set("If-Modified-Since", lastModified)
req.Header.Set("If-None-Match", etag)
return c.httpClient.Do(req)
}
var defaultClient *Client var defaultClient *Client
func searchFeedLinks(html []byte, siteurl string) ([]FeedSource, error) { func searchFeedLinks(html []byte, siteurl string) ([]FeedSource, error) {
@ -243,16 +254,37 @@ func convertItems(items []*gofeed.Item, feed storage.Feed) []storage.Item {
return result return result
} }
func listItems(f storage.Feed) ([]storage.Item, error) { func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
res, err := defaultClient.get(f.FeedLink) var res *http.Response
var err error
httpState := db.GetHTTPState(f.FeedLink)
if httpState != nil {
res, err = defaultClient.getConditional(f.FeedLink, httpState.LastModified, httpState.Etag)
} else {
res, err = defaultClient.get(f.FeedLink)
}
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer res.Body.Close() defer res.Body.Close()
if res.StatusCode == 404 { if res.StatusCode == 404 {
errmsg := fmt.Sprintf("Failed to list feed items for %s (status: 404)", f.FeedLink) errmsg := fmt.Sprintf("Failed to list feed items for %s (status: 404)", f.FeedLink)
return nil, errors.New(errmsg) return nil, errors.New(errmsg)
} }
if res.StatusCode == 304 {
return nil, nil
}
lastModified := res.Header.Get("Last-Modified")
etag := res.Header.Get("Etag")
if lastModified != "" || etag != "" {
db.SetHTTPState(f.FeedLink, storage.HTTPState{LastModified: lastModified, Etag: etag})
}
feedparser := gofeed.NewParser() feedparser := gofeed.NewParser()
feed, err := feedparser.Parse(res.Body) feed, err := feedparser.Parse(res.Body)
if err != nil { if err != nil {

2
server/server.go
View File

@ -71,7 +71,7 @@ func (h *Handler) startJobs() {
for { for {
select { select {
case feed := <-h.feedQueue: case feed := <-h.feedQueue:
items, err := listItems(feed) items, err := listItems(feed, h.db)
atomic.AddInt32(h.queueSize, -1) atomic.AddInt32(h.queueSize, -1)
if err != nil { if err != nil {
h.log.Printf("Failed to fetch %s (%d): %s", feed.FeedLink, feed.Id, err) h.log.Printf("Failed to fetch %s (%d): %s", feed.FeedLink, feed.Id, err)