james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
nkanaev 2022-01-24 12:50:52 +00:00
parent 05f5785660
commit bff7476b58
2 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/parser/feed.go
View File

@ -75,6 +75,16 @@ func Parse(r io.Reader) (*Feed, error) {
return feed, err return feed, err
} }
func ParseAndFix(r io.Reader, baseURL string) (*Feed, error) {
feed, err := Parse(r)
if err != nil {
return nil, err
}
feed.TranslateURLs(baseURL)
feed.SetMissingDatesTo(time.Now())
return feed, nil
}
func (feed *Feed) cleanup() { func (feed *Feed) cleanup() {
feed.Title = strings.TrimSpace(feed.Title) feed.Title = strings.TrimSpace(feed.Title)
feed.SiteURL = strings.TrimSpace(feed.SiteURL) feed.SiteURL = strings.TrimSpace(feed.SiteURL)

9
src/worker/crawler.go
View File

@ -9,7 +9,6 @@ import (
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
"time"
"github.com/nkanaev/yarr/src/content/scraper" "github.com/nkanaev/yarr/src/content/scraper"
"github.com/nkanaev/yarr/src/parser" "github.com/nkanaev/yarr/src/parser"
@ -50,10 +49,8 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
} }
// Try to feed into parser // Try to feed into parser
feed, err := parser.Parse(bytes.NewReader(content)) feed, err := parser.ParseAndFix(bytes.NewReader(content), candidateUrl)
if err == nil { if err == nil {
feed.TranslateURLs(candidateUrl)
feed.SetMissingDatesTo(time.Now())
result.Feed = feed result.Feed = feed
result.FeedLink = candidateUrl result.FeedLink = candidateUrl
return result, nil return result, nil
@ -192,7 +189,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
return nil, err return nil, err
} }
feed, err := parser.Parse(body) feed, err := parser.ParseAndFix(body, f.FeedLink)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -202,8 +199,6 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
if lmod != "" || etag != "" { if lmod != "" || etag != "" {
db.SetHTTPState(f.Id, lmod, etag) db.SetHTTPState(f.Id, lmod, etag)
} }
feed.TranslateURLs(f.FeedLink)
feed.SetMissingDatesTo(time.Now())
return ConvertItems(feed.Items, f), nil return ConvertItems(feed.Items, f), nil
} }