create feeds

go.mod

@@ -5,4 +5,5 @@ go 1.14
 require (
 	github.com/PuerkitoBio/goquery v1.5.1
 	github.com/mattn/go-sqlite3 v1.14.0
+	github.com/mmcdole/gofeed v1.0.0
 )

go.sum

@@ -1,14 +1,27 @@
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
 github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/mmcdole/gofeed v1.0.0 h1:PHqwr8fsEm8xarj9s53XeEAFYhRM3E9Ib7Ie766/LTE=
+github.com/mmcdole/gofeed v1.0.0/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

server/handlers.go

@@ -2,6 +2,8 @@ package server
 
 import (
 	"github.com/nkanaev/yarr/worker"
+	"github.com/nkanaev/yarr/storage"
+	"github.com/mmcdole/gofeed"
 	"net/http"
 	"encoding/json"
 	"os"
@@ -14,7 +16,6 @@ import (
 )
 
 func IndexHandler(rw http.ResponseWriter, req *http.Request) {
-	fmt.Println(os.Getwd())
 	f, err := os.Open("template/index.html")
 	if err != nil {
 		log.Fatal(err)
@@ -46,7 +47,7 @@ func FolderHandler(rw http.ResponseWriter, req *http.Request) {
 
 type NewFeed struct {
 	Url string	    `json:"url"`
-	FolderID int64 `json:"folder_id,omitempty"`
+	FolderID *int64 `json:"folder_id,omitempty"`
 }
 
 func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
@@ -57,6 +58,7 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
 			rw.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
 		feedUrl := feed.Url
 		res, err := http.Get(feedUrl)	
 		if err != nil {
@@ -73,6 +75,8 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
 			sources, err := worker.FindFeeds(res)
 			if err != nil {
 				log.Print(err)
+				rw.WriteHeader(http.StatusBadRequest)
+				return
 			}
 			if len(sources) == 0 {
 				writeJSON(rw, map[string]string{"status": "notfound"})
@@ -84,14 +88,41 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
 			} else if len(sources) == 1 {
 				feedUrl = sources[0].Url
 				fmt.Println("feedUrl:", feedUrl)
+				err = createFeed(db(req), feedUrl, 0)
+				if err == nil {
 					writeJSON(rw, map[string]string{"status": "success"})
 				}
-			fmt.Println("got html url", sources, feedUrl)
-		} else if strings.HasPrefix(contentType, "text/xml") {
-			log.Print("got rss feed")
 			}
-		log.Print(res.Header.Get("Content-Type"))
+		} else if strings.HasPrefix(contentType, "text/xml") || strings.HasPrefix(contentType, "application/xml") {
+			err = createFeed(db(req), feedUrl, 0)
+			if err == nil {
+				writeJSON(rw, map[string]string{"status": "success"})
 			}
+		} else {
+			rw.WriteHeader(http.StatusBadRequest)
+			return
+		}
+	}
+}
+
+func createFeed(s *storage.Storage, url string, folderId int64) error {
+	fmt.Println(s, url)
+	fp := gofeed.NewParser()
+	feed, err := fp.ParseURL(url)
+	if err != nil {
+		return err
+	}
+	entry := s.CreateFeed(
+		feed.Title,
+		feed.Description,
+		feed.Link,
+		feed.FeedLink,
+		"",
+		folderId,
+	)
+
+	fmt.Println("here we go", entry)
+	return nil
 }
 
 func FeedHandler(rw http.ResponseWriter, req *http.Request) {

server/server.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"regexp"
 	"net/http"
+	"github.com/nkanaev/yarr/storage"
 	"log"
 )
 
@@ -15,6 +16,7 @@ type Route struct {
 }
 
 type Handler struct {
+	db *storage.Storage
 }
 
 func p(path string, handler func(http.ResponseWriter, *http.Request)) Route {
@@ -44,25 +46,37 @@ var routes []Route = []Route{
 }
 
 func Vars(req *http.Request) map[string]string {
-	if rv := req.Context().Value(0); rv != nil {
+	if rv := req.Context().Value(ctxVars); rv != nil {
 		return rv.(map[string]string)
 	}
 	return nil
 }
 
+func db(req *http.Request) *storage.Storage {
+	if rv := req.Context().Value(ctxDB); rv != nil {
+		return rv.(*storage.Storage)
+	}
+	return nil
+}
+
+const (
+	ctxDB = 1
+	ctxVars = 2
+)
+
 func (h Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	for _, route := range routes {
 		if route.urlRegex.MatchString(req.URL.Path) {
+			ctx := context.WithValue(req.Context(), ctxDB, h.db)
 			if route.urlRegex.NumSubexp() > 0 {
 				vars := make(map[string]string)
 				matches := route.urlRegex.FindStringSubmatchIndex(req.URL.Path)
 				for i, key := range route.urlRegex.SubexpNames()[1:] {
 					vars[key] = req.URL.Path[matches[i*2+2]:matches[i*2+3]]
 				}
-				ctx := context.WithValue(req.Context(), 0, vars)
+				ctx = context.WithValue(ctx, ctxVars, vars)
-				req = req.WithContext(ctx)	
 			}
-			route.handler(rw, req)
+			route.handler(rw, req.WithContext(ctx))
 			return
 		}
 	}
@@ -80,7 +94,8 @@ func writeJSON(rw http.ResponseWriter, data interface{}) {
 }
 
 func New() *http.Server {
-	h := Handler{}
+	db, _ := storage.New()
+	h := Handler{db: db}
 	s := &http.Server{Addr: "127.0.0.1:8000", Handler: h}
 	return s
 }

storage/feed.go

@@ -13,8 +13,10 @@ type Feed struct {
 func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId int64) *Feed {
 	result, err := s.db.Exec(`
 		insert into feeds (title, description, link, feed_link, icon, folder_id) 
-		values (?, ?, ?, ?, ?, ?)`,
+		values (?, ?, ?, ?, ?, ?)
+		on conflict (feed_link) do update set folder_id=?`,
 		title, description, link, feedLink, icon, intOrNil(folderId),
+		intOrNil(folderId),
 	)
 	if err != nil {
 		return nil

storage/storage.go

@@ -20,11 +20,12 @@ create table if not exists feeds (
  title          text not null,
  description    text,
  link           text,
- feed_link      text,
+ feed_link      text not null,
  icon           text
 );
 
 create index if not exists idx_feed_folder_id on feeds(folder_id);
+create unique index if not exists idx_feed_feed_link on feeds(feed_link);
 
 create table if not exists items (
  id             string primary key,