refactoring

server/crawler.go

@@ -0,0 +1,35 @@
+package server
+
+import (
+	"net/url"
+	"net/http"
+	"github.com/PuerkitoBio/goquery"
+)
+
+type FeedSource struct {
+	Title string `json:"title"`
+	Url string `json:"url"`
+}
+
+const feedLinks = `link[type='application/rss+xml'],link[type='application/atom+xml']`
+
+
+func FindFeeds(r *http.Response) ([]FeedSource, error) {
+	sources := make([]FeedSource, 0, 0)
+	doc, err := goquery.NewDocumentFromResponse(r)
+	if err != nil {
+		return sources, err
+	}
+	doc.Find(feedLinks).Each(func(i int, s *goquery.Selection) {
+		if href, ok := s.Attr("href"); ok {
+			feedUrl, err := url.Parse(href)
+			if err != nil {
+				return
+			}
+			title := s.AttrOr("title", "")
+			url := doc.Url.ResolveReference(feedUrl).String()
+			sources = append(sources, FeedSource{Title: title, Url: url})
+		}
+	})
+	return sources, nil
+}

server/handlers.go

@@ -1,7 +1,6 @@
 package server
 
 import (
-	"github.com/nkanaev/yarr/worker"
 	"github.com/nkanaev/yarr/storage"
 	"github.com/mmcdole/gofeed"
 	"net/http"
@@ -137,7 +136,7 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
 
 		contentType := res.Header.Get("Content-Type")
 		if strings.HasPrefix(contentType, "text/html") || contentType == "" {
-			sources, err := worker.FindFeeds(res)
+			sources, err := FindFeeds(res)
 			if err != nil {
 				log.Print(err)
 				rw.WriteHeader(http.StatusBadRequest)