feed finder

go.mod

@@ -2,4 +2,7 @@ module github.com/nkanaev/yarr
 
 go 1.14
 
-require github.com/mattn/go-sqlite3 v1.14.0
+require (
+	github.com/PuerkitoBio/goquery v1.5.1
+	github.com/mattn/go-sqlite3 v1.14.0
+)

go.sum

@@ -1,10 +1,13 @@
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"github.com/nkanaev/yarr/storage"
+	"github.com/nkanaev/yarr/worker"
 	"log"
 )
 
@@ -67,4 +68,8 @@ func main() {
 	log.Print(store.CreateItems(items))
 	log.Print(store.ListItems())
 	*/
+	/*
+	log.Print(worker.FindFeeds("https://horriblesubs.info/"))
+	log.Print(worker.FindFeeds("http://daringfireball.net/"))
+	*/
 }

worker/finder.go

@@ -0,0 +1,37 @@
+package worker
+
+import (
+	"log"
+	//"net/http"
+	"net/url"
+	"github.com/PuerkitoBio/goquery"
+)
+
+type FeedSource struct {
+	Title string
+	Url *url.URL
+}
+
+
+func FindFeeds(u string) []FeedSource {
+	doc, err := goquery.NewDocument(u)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Print(doc.Url)
+	// Find the review items
+	sources := make([]FeedSource, 0, 0)
+	doc.Find("link[type='application/rss+xml'],link[type='application/atom+xml']").Each(func(i int, s *goquery.Selection) {
+		if href, ok := s.Attr("href"); ok {
+			feedUrl, feedErr := url.Parse(href)
+			if feedErr != nil {
+				log.Fatal(err)
+			}
+			title := s.AttrOr("title", "")
+			feedUrl = doc.Url.ResolveReference(feedUrl)
+			sources = append(sources, FeedSource{Title: title, Url: feedUrl})
+		}
+	})
+	return sources
+}