reorganizing content-related packages

src/content/scraper/finder.go

@@ -0,0 +1,97 @@
+package scraper
+
+import (
+	"strings"
+
+	"github.com/nkanaev/yarr/src/content/htmlutil"
+	"golang.org/x/net/html"
+)
+
+func FindFeeds(body string, base string) map[string]string {
+	candidates := make(map[string]string)
+
+	doc, err := html.Parse(strings.NewReader(body))
+	if err != nil {
+		return candidates
+	}
+
+	// find direct links
+	// css: link[type=application/atom+xml]
+	linkTypes := []string{"application/atom+xml", "application/rss+xml", "application/json"}
+	isFeedLink := func(n *html.Node) bool {
+		if n.Type == html.ElementNode && n.Data == "link" {
+			t := htmlutil.Attr(n, "type")
+			for _, tt := range linkTypes {
+				if tt == t {
+					return true
+				}
+			}
+		}
+		return false
+	}
+	for _, node := range htmlutil.FindNodes(doc, isFeedLink) {
+		href := htmlutil.Attr(node, "href")
+		name := htmlutil.Attr(node, "title")
+		link := htmlutil.AbsoluteUrl(href, base)
+		if link != "" {
+			candidates[link] = name
+		}
+	}
+
+	// guess by hyperlink properties
+	if len(candidates) == 0 {
+		// css: a[href="feed"]
+		// css: a:contains("rss")
+		feedHrefs := []string{"feed", "feed.xml", "rss.xml", "atom.xml"}
+		feedTexts := []string{"rss", "feed"}
+		isFeedHyperLink := func(n *html.Node) bool {
+			if n.Type == html.ElementNode && n.Data == "a" {
+				href := strings.Trim(htmlutil.Attr(n, "href"), "/")
+				for _, feedHref := range feedHrefs {
+					if strings.HasSuffix(href, feedHref) {
+						return true
+					}
+				}
+				text := htmlutil.Text(n)
+				for _, feedText := range feedTexts {
+					if strings.EqualFold(text, feedText) {
+						return true
+					}
+				}
+			}
+			return false
+		}
+		for _, node := range htmlutil.FindNodes(doc, isFeedHyperLink) {
+			href := htmlutil.Attr(node, "href")
+			link := htmlutil.AbsoluteUrl(href, base)
+			if link != "" {
+				candidates[link] = ""
+			}
+		}
+	}
+
+	return candidates
+}
+
+func FindIcons(body string, base string) []string {
+	icons := make([]string, 0)
+
+	doc, err := html.Parse(strings.NewReader(body))
+	if err != nil {
+		return icons
+	}
+
+	// css: link[rel=icon]
+	isLink := func(n *html.Node) bool {
+		return n.Type == html.ElementNode && n.Data == "link"
+	}
+	for _, node := range htmlutil.FindNodes(doc, isLink) {
+		rels := strings.Split(htmlutil.Attr(node, "rel"), " ")
+		for _, rel := range rels {
+			if strings.EqualFold(rel, "icon") {
+				icons = append(icons, htmlutil.AbsoluteUrl(htmlutil.Attr(node, "href"), base))
+			}
+		}
+	}
+	return icons
+}