james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-09-13 18:00:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
nkanaev
2022-01-24 12:50:52 +00:00
parent 05f5785660
commit bff7476b58
2 changed files with 12 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/worker/crawler.go
View File

@@ -9,7 +9,6 @@ import (
"net/http"
"net/url"
"strings"
"time"
"github.com/nkanaev/yarr/src/content/scraper"
"github.com/nkanaev/yarr/src/parser"
@@ -50,10 +49,8 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
}
// Try to feed into parser
feed, err := parser.Parse(bytes.NewReader(content))
feed, err := parser.ParseAndFix(bytes.NewReader(content), candidateUrl)
if err == nil {
feed.TranslateURLs(candidateUrl)
feed.SetMissingDatesTo(time.Now())
result.Feed = feed
result.FeedLink = candidateUrl
return result, nil
@@ -192,7 +189,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
return nil, err
}
feed, err := parser.Parse(body)
feed, err := parser.ParseAndFix(body, f.FeedLink)
if err != nil {
return nil, err
}
@@ -202,8 +199,6 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
if lmod != "" || etag != "" {
db.SetHTTPState(f.Id, lmod, etag)
}
feed.TranslateURLs(f.FeedLink)
feed.SetMissingDatesTo(time.Now())
return ConvertItems(feed.Items, f), nil
}