diff --git a/go.mod b/go.mod index d8484d3..1ee2e93 100644 --- a/go.mod +++ b/go.mod @@ -5,4 +5,5 @@ go 1.14 require ( github.com/PuerkitoBio/goquery v1.5.1 github.com/mattn/go-sqlite3 v1.14.0 + github.com/mmcdole/gofeed v1.0.0 ) diff --git a/go.sum b/go.sum index d5013b4..def448a 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,27 @@ +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mmcdole/gofeed v1.0.0 h1:PHqwr8fsEm8xarj9s53XeEAFYhRM3E9Ib7Ie766/LTE= +github.com/mmcdole/gofeed v1.0.0/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/server/handlers.go b/server/handlers.go index 1c443f0..1a04df3 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -2,6 +2,8 @@ package server import ( "github.com/nkanaev/yarr/worker" + "github.com/nkanaev/yarr/storage" + "github.com/mmcdole/gofeed" "net/http" "encoding/json" "os" @@ -14,7 +16,6 @@ import ( ) func IndexHandler(rw http.ResponseWriter, req *http.Request) { - fmt.Println(os.Getwd()) f, err := os.Open("template/index.html") if err != nil { log.Fatal(err) @@ -45,8 +46,8 @@ func FolderHandler(rw http.ResponseWriter, req *http.Request) { } type NewFeed struct { - Url string `json:"url"` - FolderID int64 `json:"folder_id,omitempty"` + Url string `json:"url"` + FolderID *int64 `json:"folder_id,omitempty"` } func FeedListHandler(rw http.ResponseWriter, req *http.Request) { @@ -57,6 +58,7 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) { rw.WriteHeader(http.StatusBadRequest) return } + feedUrl := feed.Url res, err := http.Get(feedUrl) if err != nil { @@ -73,6 +75,8 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) { sources, err := worker.FindFeeds(res) if err != nil { log.Print(err) + rw.WriteHeader(http.StatusBadRequest) + return } if len(sources) == 0 { writeJSON(rw, map[string]string{"status": "notfound"}) @@ -84,15 +88,42 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) { } else if len(sources) == 1 { feedUrl = sources[0].Url fmt.Println("feedUrl:", feedUrl) + err = createFeed(db(req), feedUrl, 0) + if err == nil { + writeJSON(rw, map[string]string{"status": "success"}) + } + } + } else if strings.HasPrefix(contentType, "text/xml") || strings.HasPrefix(contentType, "application/xml") { + err = createFeed(db(req), feedUrl, 0) + if err == nil { writeJSON(rw, map[string]string{"status": "success"}) } - fmt.Println("got html url", sources, feedUrl) - } else if strings.HasPrefix(contentType, "text/xml") { - log.Print("got rss feed") + } else { + rw.WriteHeader(http.StatusBadRequest) + return } - log.Print(res.Header.Get("Content-Type")) } } +func createFeed(s *storage.Storage, url string, folderId int64) error { + fmt.Println(s, url) + fp := gofeed.NewParser() + feed, err := fp.ParseURL(url) + if err != nil { + return err + } + entry := s.CreateFeed( + feed.Title, + feed.Description, + feed.Link, + feed.FeedLink, + "", + folderId, + ) + + fmt.Println("here we go", entry) + return nil +} + func FeedHandler(rw http.ResponseWriter, req *http.Request) { } diff --git a/server/server.go b/server/server.go index 2baaaa6..cb872a0 100644 --- a/server/server.go +++ b/server/server.go @@ -5,6 +5,7 @@ import ( "context" "regexp" "net/http" + "github.com/nkanaev/yarr/storage" "log" ) @@ -15,6 +16,7 @@ type Route struct { } type Handler struct { + db *storage.Storage } func p(path string, handler func(http.ResponseWriter, *http.Request)) Route { @@ -44,25 +46,37 @@ var routes []Route = []Route{ } func Vars(req *http.Request) map[string]string { - if rv := req.Context().Value(0); rv != nil { + if rv := req.Context().Value(ctxVars); rv != nil { return rv.(map[string]string) } return nil } +func db(req *http.Request) *storage.Storage { + if rv := req.Context().Value(ctxDB); rv != nil { + return rv.(*storage.Storage) + } + return nil +} + +const ( + ctxDB = 1 + ctxVars = 2 +) + func (h Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { for _, route := range routes { if route.urlRegex.MatchString(req.URL.Path) { + ctx := context.WithValue(req.Context(), ctxDB, h.db) if route.urlRegex.NumSubexp() > 0 { vars := make(map[string]string) matches := route.urlRegex.FindStringSubmatchIndex(req.URL.Path) for i, key := range route.urlRegex.SubexpNames()[1:] { vars[key] = req.URL.Path[matches[i*2+2]:matches[i*2+3]] } - ctx := context.WithValue(req.Context(), 0, vars) - req = req.WithContext(ctx) + ctx = context.WithValue(ctx, ctxVars, vars) } - route.handler(rw, req) + route.handler(rw, req.WithContext(ctx)) return } } @@ -80,7 +94,8 @@ func writeJSON(rw http.ResponseWriter, data interface{}) { } func New() *http.Server { - h := Handler{} + db, _ := storage.New() + h := Handler{db: db} s := &http.Server{Addr: "127.0.0.1:8000", Handler: h} return s } diff --git a/storage/feed.go b/storage/feed.go index c5451cd..0028bf2 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -13,8 +13,10 @@ type Feed struct { func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId int64) *Feed { result, err := s.db.Exec(` insert into feeds (title, description, link, feed_link, icon, folder_id) - values (?, ?, ?, ?, ?, ?)`, + values (?, ?, ?, ?, ?, ?) + on conflict (feed_link) do update set folder_id=?`, title, description, link, feedLink, icon, intOrNil(folderId), + intOrNil(folderId), ) if err != nil { return nil diff --git a/storage/storage.go b/storage/storage.go index 5e6dab9..e426f82 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -20,11 +20,12 @@ create table if not exists feeds ( title text not null, description text, link text, - feed_link text, + feed_link text not null, icon text ); create index if not exists idx_feed_folder_id on feeds(folder_id); +create unique index if not exists idx_feed_feed_link on feeds(feed_link); create table if not exists items ( id string primary key,