diff --git a/src/feed/atom.go b/src/parser/atom.go similarity index 99% rename from src/feed/atom.go rename to src/parser/atom.go index c33af22..2aaf745 100644 --- a/src/feed/atom.go +++ b/src/parser/atom.go @@ -1,5 +1,5 @@ // Atom 1.0 parser -package feed +package parser import ( "encoding/xml" diff --git a/src/feed/atom_test.go b/src/parser/atom_test.go similarity index 99% rename from src/feed/atom_test.go rename to src/parser/atom_test.go index cd4c332..bd14d79 100644 --- a/src/feed/atom_test.go +++ b/src/parser/atom_test.go @@ -1,4 +1,4 @@ -package feed +package parser import ( "reflect" diff --git a/src/feed/feed.go b/src/parser/feed.go similarity index 99% rename from src/feed/feed.go rename to src/parser/feed.go index 0e4d656..1339a78 100644 --- a/src/feed/feed.go +++ b/src/parser/feed.go @@ -1,4 +1,4 @@ -package feed +package parser import ( "bytes" diff --git a/src/feed/feed_test.go b/src/parser/feed_test.go similarity index 98% rename from src/feed/feed_test.go rename to src/parser/feed_test.go index 3d473ba..a333e00 100644 --- a/src/feed/feed_test.go +++ b/src/parser/feed_test.go @@ -1,4 +1,4 @@ -package feed +package parser import ( "reflect" diff --git a/src/feed/json.go b/src/parser/json.go similarity index 99% rename from src/feed/json.go rename to src/parser/json.go index 5af1684..fd63bbd 100644 --- a/src/feed/json.go +++ b/src/parser/json.go @@ -1,5 +1,5 @@ // JSON 1.0 parser -package feed +package parser import ( "encoding/json" diff --git a/src/feed/json_test.go b/src/parser/json_test.go similarity index 98% rename from src/feed/json_test.go rename to src/parser/json_test.go index 9957398..f94fd1b 100644 --- a/src/feed/json_test.go +++ b/src/parser/json_test.go @@ -1,4 +1,4 @@ -package feed +package parser import ( "reflect" diff --git a/src/feed/models.go b/src/parser/models.go similarity index 93% rename from src/feed/models.go rename to src/parser/models.go index dfcaaf4..7d2a28f 100644 --- a/src/feed/models.go +++ b/src/parser/models.go @@ -1,4 +1,4 @@ -package feed +package parser import "time" diff --git a/src/feed/rdf.go b/src/parser/rdf.go similarity index 98% rename from src/feed/rdf.go rename to src/parser/rdf.go index 5f6d7f8..276ad7a 100644 --- a/src/feed/rdf.go +++ b/src/parser/rdf.go @@ -1,7 +1,7 @@ // Parser for RSS versions: // - 0.90 // - 1.0 -package feed +package parser import ( "encoding/xml" diff --git a/src/feed/rdf_test.go b/src/parser/rdf_test.go similarity index 98% rename from src/feed/rdf_test.go rename to src/parser/rdf_test.go index 97ff7b1..3d3e9ed 100644 --- a/src/feed/rdf_test.go +++ b/src/parser/rdf_test.go @@ -1,4 +1,4 @@ -package feed +package parser import ( "reflect" diff --git a/src/feed/rss.go b/src/parser/rss.go similarity index 99% rename from src/feed/rss.go rename to src/parser/rss.go index e11eda2..8cb88cd 100644 --- a/src/feed/rss.go +++ b/src/parser/rss.go @@ -2,7 +2,7 @@ // - 0.91 netscape // - 0.91 userland // - 2.0 -package feed +package parser import ( "encoding/xml" diff --git a/src/feed/rss_test.go b/src/parser/rss_test.go similarity index 98% rename from src/feed/rss_test.go rename to src/parser/rss_test.go index cad4d14..89896ca 100644 --- a/src/feed/rss_test.go +++ b/src/parser/rss_test.go @@ -1,4 +1,4 @@ -package feed +package parser import ( "reflect" diff --git a/src/feed/utils.go b/src/parser/utils.go similarity index 99% rename from src/feed/utils.go rename to src/parser/utils.go index 38b626e..545cded 100644 --- a/src/feed/utils.go +++ b/src/parser/utils.go @@ -1,4 +1,4 @@ -package feed +package parser import ( "encoding/xml" diff --git a/src/crawler/finder.go b/src/scraper/finder.go similarity index 99% rename from src/crawler/finder.go rename to src/scraper/finder.go index 8fe17d2..674cd53 100644 --- a/src/crawler/finder.go +++ b/src/scraper/finder.go @@ -1,4 +1,4 @@ -package crawler +package scraper import ( "strings" diff --git a/src/crawler/finder_test.go b/src/scraper/finder_test.go similarity index 99% rename from src/crawler/finder_test.go rename to src/scraper/finder_test.go index 0fa6399..3b48333 100644 --- a/src/crawler/finder_test.go +++ b/src/scraper/finder_test.go @@ -1,4 +1,4 @@ -package crawler +package scraper import ( "reflect" diff --git a/src/crawler/utils.go b/src/scraper/utils.go similarity index 98% rename from src/crawler/utils.go rename to src/scraper/utils.go index 0381fe0..dba54b3 100644 --- a/src/crawler/utils.go +++ b/src/scraper/utils.go @@ -1,4 +1,4 @@ -package crawler +package scraper import ( "net/url" diff --git a/src/worker/crawler.go b/src/worker/crawler.go index b9163b8..f922e63 100644 --- a/src/worker/crawler.go +++ b/src/worker/crawler.go @@ -11,8 +11,8 @@ import ( "net/url" "time" - "github.com/nkanaev/yarr/src/crawler" - feedparser "github.com/nkanaev/yarr/src/feed" + "github.com/nkanaev/yarr/src/scraper" + "github.com/nkanaev/yarr/src/parser" "github.com/nkanaev/yarr/src/storage" "golang.org/x/net/html/charset" ) @@ -55,13 +55,13 @@ var defaultClient *Client func searchFeedLinks(html []byte, siteurl string) ([]FeedSource, error) { sources := make([]FeedSource, 0, 0) - for url, title := range crawler.FindFeeds(string(html), siteurl) { + for url, title := range scraper.FindFeeds(string(html), siteurl) { sources = append(sources, FeedSource{Title: title, Url: url}) } return sources, nil } -func DiscoverFeed(candidateUrl string) (*feedparser.Feed, string, *[]FeedSource, error) { +func DiscoverFeed(candidateUrl string) (*parser.Feed, string, *[]FeedSource, error) { // Query URL res, err := defaultClient.get(candidateUrl) if err != nil { @@ -78,7 +78,7 @@ func DiscoverFeed(candidateUrl string) (*feedparser.Feed, string, *[]FeedSource, } // Try to feed into parser - feed, err := feedparser.Parse(bytes.NewReader(content)) + feed, err := parser.Parse(bytes.NewReader(content)) if err == nil { /* // WILD: feeds may not always have link to themselves @@ -141,7 +141,7 @@ func FindFavicon(websiteUrl, feedUrl string) (*[]byte, error) { if err != nil { return nil, err } - candidateUrls = append(candidateUrls, crawler.FindIcons(string(body), websiteUrl)...) + candidateUrls = append(candidateUrls, scraper.FindIcons(string(body), websiteUrl)...) if c := favicon(websiteUrl); len(c) != 0 { candidateUrls = append(candidateUrls, c) } @@ -176,7 +176,7 @@ func FindFavicon(websiteUrl, feedUrl string) (*[]byte, error) { return nil, nil } -func ConvertItems(items []feedparser.Item, feed storage.Feed) []storage.Item { +func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item { result := make([]storage.Item, len(items)) for i, item := range items { item := item @@ -237,7 +237,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) { if err != nil { return nil, fmt.Errorf("failed to init response body: %s", err) } - feed, err := feedparser.Parse(body) + feed, err := parser.Parse(body) if err != nil { return nil, fmt.Errorf("failed to parse: %s", err) }