diff --git a/src/server/auth/middleware.go b/src/server/auth/middleware.go index 4aca787..b559be9 100644 --- a/src/server/auth/middleware.go +++ b/src/server/auth/middleware.go @@ -12,7 +12,7 @@ type Middleware struct { Username string Password string BasePath string - Public string + Public []string } func unsafeMethod(method string) bool { @@ -20,9 +20,11 @@ func unsafeMethod(method string) bool { } func (m *Middleware) Handler(c *router.Context) { - if strings.HasPrefix(c.Req.URL.Path, m.BasePath+m.Public) { - c.Next() - return + for _, path := range m.Public { + if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) { + c.Next() + return + } } if IsAuthenticated(c.Req, m.Username, m.Password) { c.Next() diff --git a/src/server/fever.go b/src/server/fever.go new file mode 100644 index 0000000..0f17a1a --- /dev/null +++ b/src/server/fever.go @@ -0,0 +1,385 @@ +package server + +import ( + "crypto/md5" + "encoding/base64" + "fmt" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/nkanaev/yarr/src/server/auth" + "github.com/nkanaev/yarr/src/server/router" + "github.com/nkanaev/yarr/src/storage" +) + +type FeverGroup struct { + ID int64 `json:"id"` + Title string `json:"title"` +} + +type FeverFeedsGroup struct { + GroupID int64 `json:"group_id"` + FeedIDs string `json:"feed_ids"` +} + +type FeverFeed struct { + ID int64 `json:"id"` + FaviconID int64 `json:"favicon_id"` + Title string `json:"title"` + Url string `json:"url"` + SiteUrl string `json:"site_url"` + IsSpark int `json:"is_spark"` + LastUpdated int64 `json:"last_updated_on_time"` +} + +type FeverItem struct { + ID int64 `json:"id"` + FeedID int64 `json:"feed_id"` + Title string `json:"title"` + Author string `json:"author"` + HTML string `json:"html"` + Url string `json:"url"` + IsSaved int `json:"is_saved"` + IsRead int `json:"is_read"` + CreatedAt int64 `json:"created_on_time"` +} + +type FeverFavicon struct { + ID int64 `json:"id"` + Data string `json:"data"` +} + +func writeFeverJSON(c *router.Context, data map[string]interface{}, lastRefreshed int64) { + data["api_version"] = 1 + data["auth"] = 1 + data["last_refreshed_on_time"] = lastRefreshed + c.JSON(http.StatusOK, data) +} + +func getLastRefreshedOnTime(httpStates map[int64]storage.HTTPState) int64 { + if len(httpStates) == 0 { + return 0 + } + + var lastRefreshed int64 + for _, state := range httpStates { + if state.LastRefreshed.Unix() > lastRefreshed { + lastRefreshed = state.LastRefreshed.Unix() + } + } + return lastRefreshed +} + +func (s *Server) feverAuth(c *router.Context) bool { + if s.Username != "" && s.Password != "" { + apiKey := c.Req.FormValue("api_key") + md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password))) + hexMD5HashValue := fmt.Sprintf("%x", md5HashValue[:]) + if auth.StringsEqual(apiKey, hexMD5HashValue) { + return false + } + } + return true +} + +func formHasValue(values url.Values, value string) bool { + if _, ok := values[value]; ok { + return true + } + return false +} + +func (s *Server) handleFever(c *router.Context) { + c.Req.ParseForm() + if !s.feverAuth(c) { + c.JSON(http.StatusOK, map[string]interface{}{ + "api_version": 1, + "auth": 0, + "last_refreshed_on_time": 0, + }) + return + } + + switch { + case formHasValue(c.Req.Form, "groups"): + s.feverGroupsHandler(c) + case formHasValue(c.Req.Form, "feeds"): + s.feverFeedsHandler(c) + case formHasValue(c.Req.Form, "unread_item_ids"): + s.feverUnreadItemIDsHandler(c) + case formHasValue(c.Req.Form, "saved_item_ids"): + s.feverSavedItemIDsHandler(c) + case formHasValue(c.Req.Form, "favicons"): + s.feverFaviconsHandler(c) + case formHasValue(c.Req.Form, "items"): + s.feverItemsHandler(c) + case formHasValue(c.Req.Form, "links"): + s.feverLinksHandler(c) + case formHasValue(c.Req.Form, "mark"): + s.feverMarkHandler(c) + default: + c.JSON(http.StatusOK, map[string]interface{}{ + "api_version": 1, + "auth": 1, + "last_refreshed_on_time": getLastRefreshedOnTime(s.db.ListHTTPStates()), + }) + } +} + +func joinInts(values []int64) string { + var result strings.Builder + for i, val := range values { + fmt.Fprintf(&result, "%d", val) + if i != len(values)-1 { + result.WriteString(",") + } + } + return result.String() +} + +func feedGroups(db *storage.Storage) []*FeverFeedsGroup { + feeds := db.ListFeeds() + + groupFeeds := make(map[int64][]int64) + for _, feed := range feeds { + if feed.FolderId == nil { + continue + } + groupFeeds[*feed.FolderId] = append(groupFeeds[*feed.FolderId], feed.Id) + } + result := make([]*FeverFeedsGroup, 0) + for groupId, feedIds := range groupFeeds { + result = append(result, &FeverFeedsGroup{ + GroupID: groupId, + FeedIDs: joinInts(feedIds), + }) + } + return result +} + +func (s *Server) feverGroupsHandler(c *router.Context) { + folders := s.db.ListFolders() + groups := make([]*FeverGroup, len(folders)) + for i, folder := range folders { + groups[i] = &FeverGroup{ID: folder.Id, Title: folder.Title} + } + writeFeverJSON(c, map[string]interface{}{ + "groups": groups, + "feeds_groups": feedGroups(s.db), + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) +} + +func (s *Server) feverFeedsHandler(c *router.Context) { + feeds := s.db.ListFeeds() + httpStates := s.db.ListHTTPStates() + + feverFeeds := make([]*FeverFeed, len(feeds)) + for i, feed := range feeds { + var lastUpdated int64 + if state, ok := httpStates[feed.Id]; ok { + lastUpdated = state.LastRefreshed.Unix() + } + feverFeeds[i] = &FeverFeed{ + ID: feed.Id, + FaviconID: feed.Id, + Title: feed.Title, + Url: feed.FeedLink, + SiteUrl: feed.Link, + IsSpark: 0, + LastUpdated: lastUpdated, + } + } + writeFeverJSON(c, map[string]interface{}{ + "feeds": feverFeeds, + "feeds_groups": feedGroups(s.db), + }, getLastRefreshedOnTime(httpStates)) +} + +func (s *Server) feverFaviconsHandler(c *router.Context) { + feeds := s.db.ListFeeds() + favicons := make([]*FeverFavicon, len(feeds)) + for i, feed := range feeds { + data := "data:image/gif;base64,R0lGODlhAQABAAAAACw=" + if feed.HasIcon { + icon := s.db.GetFeed(feed.Id).Icon + data = fmt.Sprintf( + "data:%s;base64,%s", + http.DetectContentType(*icon), + base64.StdEncoding.EncodeToString(*icon), + ) + } + favicons[i] = &FeverFavicon{ID: feed.Id, Data: data} + } + + writeFeverJSON(c, map[string]interface{}{ + "favicons": favicons, + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) +} + +// for memory pressure reasons, we only return a limited number of items +// documented at https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md#items +const listLimit = 50 + +func (s *Server) feverItemsHandler(c *router.Context) { + filter := storage.ItemFilter{} + query := c.Req.URL.Query() + + switch { + case query.Get("with_ids") != "": + ids := make([]int64, 0) + for _, idstr := range strings.Split(query.Get("with_ids"), ",") { + if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil { + ids = append(ids, idnum) + } + } + filter.IDs = &ids + case query.Get("since_id") != "": + idstr := query.Get("since_id") + if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil { + filter.SinceID = &idnum + } + case query.Get("max_id") != "": + idstr := query.Get("max_id") + if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil { + filter.MaxID = &idnum + } + } + + items := s.db.ListItems(filter, listLimit, true, true) + + feverItems := make([]FeverItem, len(items)) + for i, item := range items { + date := item.Date + time := date.Unix() + + isSaved := 0 + if item.Status == storage.STARRED { + isSaved = 1 + } + isRead := 0 + if item.Status == storage.READ { + isRead = 1 + } + feverItems[i] = FeverItem{ + ID: item.Id, + FeedID: item.FeedId, + Title: item.Title, + Author: "", + HTML: item.Content, + Url: item.Link, + IsSaved: isSaved, + IsRead: isRead, + CreatedAt: time, + } + } + + writeFeverJSON(c, map[string]interface{}{ + "items": feverItems, + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) +} + +func (s *Server) feverLinksHandler(c *router.Context) { + writeFeverJSON(c, map[string]interface{}{ + "links": make([]interface{}, 0), + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) +} + +func (s *Server) feverUnreadItemIDsHandler(c *router.Context) { + status := storage.UNREAD + itemIds := make([]int64, 0) + + itemFilter := storage.ItemFilter{ + Status: &status, + } + for { + items := s.db.ListItems(itemFilter, listLimit, true, false) + if len(items) == 0 { + break + } + for _, item := range items { + itemIds = append(itemIds, item.Id) + } + itemFilter.After = &items[len(items)-1].Id + } + writeFeverJSON(c, map[string]interface{}{ + "unread_item_ids": joinInts(itemIds), + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) +} + +func (s *Server) feverSavedItemIDsHandler(c *router.Context) { + status := storage.STARRED + itemIds := make([]int64, 0) + + itemFilter := storage.ItemFilter{ + Status: &status, + } + for { + items := s.db.ListItems(itemFilter, listLimit, true, false) + if len(items) == 0 { + break + } + for _, item := range items { + itemIds = append(itemIds, item.Id) + } + itemFilter.After = &items[len(items)-1].Id + } + writeFeverJSON(c, map[string]interface{}{ + "saved_item_ids": joinInts(itemIds), + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) +} + +func (s *Server) feverMarkHandler(c *router.Context) { + id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64) + if err != nil { + log.Print("invalid id:", err) + return + } + + switch c.Req.Form.Get("mark") { + case "item": + var status storage.ItemStatus + switch c.Req.Form.Get("as") { + case "read": + status = storage.READ + case "unread": + status = storage.UNREAD + case "saved": + status = storage.STARRED + case "unsaved": + status = storage.READ + default: + c.Out.WriteHeader(http.StatusBadRequest) + return + } + s.db.UpdateItemStatus(id, status) + case "feed": + if c.Req.Form.Get("as") != "read" { + c.Out.WriteHeader(http.StatusBadRequest) + } + markFilter := storage.MarkFilter{FeedID: &id} + x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) + if x > 0 { + before := time.Unix(x, 0) + markFilter.Before = &before + } + s.db.MarkItemsRead(markFilter) + case "group": + if c.Req.Form.Get("as") != "read" { + c.Out.WriteHeader(http.StatusBadRequest) + } + markFilter := storage.MarkFilter{FolderID: &id} + x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) + if x > 0 { + before := time.Unix(x, 0) + markFilter.Before = &before + } + s.db.MarkItemsRead(markFilter) + default: + c.Out.WriteHeader(http.StatusBadRequest) + return + } +} diff --git a/src/server/routes.go b/src/server/routes.go index 8166328..073464f 100644 --- a/src/server/routes.go +++ b/src/server/routes.go @@ -34,7 +34,7 @@ func (s *Server) handler() http.Handler { BasePath: s.BasePath, Username: s.Username, Password: s.Password, - Public: "/static", + Public: []string{"/static", "/fever"}, } r.Use(a.Handler) } @@ -57,6 +57,7 @@ func (s *Server) handler() http.Handler { r.For("/opml/export", s.handleOPMLExport) r.For("/page", s.handlePageCrawl) r.For("/logout", s.handleLogout) + r.For("/fever/", s.handleFever) return r } @@ -364,7 +365,7 @@ func (s *Server) handleItemList(c *router.Context) { } newestFirst := query.Get("oldest_first") != "true" - items := s.db.ListItems(filter, perPage+1, newestFirst) + items := s.db.ListItems(filter, perPage+1, newestFirst, false) hasMore := false if len(items) == perPage+1 { hasMore = true diff --git a/src/storage/item.go b/src/storage/item.go index aff7c0f..033878a 100644 --- a/src/storage/item.go +++ b/src/storage/item.go @@ -62,11 +62,17 @@ type ItemFilter struct { Status *ItemStatus Search *string After *int64 + IDs *[]int64 + SinceID *int64 + MaxID *int64 + Before *time.Time } type MarkFilter struct { FolderID *int64 FeedID *int64 + + Before *time.Time } func (s *Storage) CreateItems(items []Item) bool { @@ -140,6 +146,28 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = ?)", compare)) args = append(args, *filter.After) } + if filter.IDs != nil && len(*filter.IDs) > 0 { + qmarks := make([]string, len(*filter.IDs)) + idargs := make([]interface{}, len(*filter.IDs)) + for i, id := range *filter.IDs { + qmarks[i] = "?" + idargs[i] = id + } + cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")") + args = append(args, idargs...) + } + if filter.SinceID != nil { + cond = append(cond, "i.id > ?") + args = append(args, filter.SinceID) + } + if filter.MaxID != nil { + cond = append(cond, "i.id < ?") + args = append(args, filter.MaxID) + } + if filter.Before != nil { + cond = append(cond, "i.date < ?") + args = append(args, filter.Before) + } predicate := "1" if len(cond) > 0 { @@ -149,7 +177,7 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac return predicate, args } -func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []Item { +func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item { predicate, args := listQueryPredicate(filter, newestFirst) result := make([]Item, 0, 0) @@ -157,17 +185,23 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It if !newestFirst { order = "date asc, id asc" } + if filter.IDs != nil || filter.SinceID != nil || filter.MaxID != nil { + order = "i.id asc" + } + selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url" + if withContent { + selectCols += ", i.content" + } else { + selectCols += ", '' as content" + } query := fmt.Sprintf(` - select - i.id, i.guid, i.feed_id, - i.title, i.link, i.date, - i.status, i.image, i.podcast_url + select %s from items i where %s order by %s limit %d - `, predicate, order, limit) + `, selectCols, predicate, order, limit) rows, err := s.db.Query(query, args...) if err != nil { log.Print(err) @@ -178,7 +212,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It err = rows.Scan( &x.Id, &x.GUID, &x.FeedId, &x.Title, &x.Link, &x.Date, - &x.Status, &x.ImageURL, &x.AudioURL, + &x.Status, &x.ImageURL, &x.AudioURL, &x.Content, ) if err != nil { log.Print(err) @@ -214,7 +248,11 @@ func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool { } func (s *Storage) MarkItemsRead(filter MarkFilter) bool { - predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID}, false) + predicate, args := listQueryPredicate(ItemFilter{ + FolderID: filter.FolderID, + FeedID: filter.FeedID, + Before: filter.Before, + }, false) query := fmt.Sprintf(` update items as i set status = %d where %s and i.status != %d diff --git a/src/storage/item_test.go b/src/storage/item_test.go index dd3cb80..e227eba 100644 --- a/src/storage/item_test.go +++ b/src/storage/item_test.go @@ -104,7 +104,7 @@ func TestListItems(t *testing.T) { // filter by folder_id - have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false)) + have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false)) want := []string{"item111", "item112", "item113", "item121", "item122"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -112,7 +112,7 @@ func TestListItems(t *testing.T) { t.Fail() } - have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false)) want = []string{"item211", "item212"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -122,7 +122,7 @@ func TestListItems(t *testing.T) { // filter by feed_id - have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false)) want = []string{"item111", "item112", "item113"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -130,7 +130,7 @@ func TestListItems(t *testing.T) { t.Fail() } - have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false)) want = []string{"item011", "item012", "item013"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -141,7 +141,7 @@ func TestListItems(t *testing.T) { // filter by status var starred ItemStatus = STARRED - have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false)) want = []string{"item113", "item212", "item013"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -150,7 +150,7 @@ func TestListItems(t *testing.T) { } var unread ItemStatus = UNREAD - have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false)) want = []string{"item111", "item121", "item011"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -160,7 +160,7 @@ func TestListItems(t *testing.T) { // limit - have = getItemGuids(db.ListItems(ItemFilter{}, 2, false)) + have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false)) want = []string{"item111", "item112"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -171,7 +171,7 @@ func TestListItems(t *testing.T) { // filter by search db.SyncSearch() search1 := "title111" - have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true)) + have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false)) want = []string{"item111"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -180,7 +180,7 @@ func TestListItems(t *testing.T) { } // sort by date - have = getItemGuids(db.ListItems(ItemFilter{}, 4, true)) + have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false)) want = []string{"item013", "item012", "item011", "item212"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -197,7 +197,7 @@ func TestListItemsPaginated(t *testing.T) { item121 := getItem(db, "item121") // all, newest first - have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true)) + have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false)) want := []string{"item011", "item212", "item211"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -207,7 +207,7 @@ func TestListItemsPaginated(t *testing.T) { // unread, newest first unread := UNREAD - have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true)) + have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false)) want = []string{"item011", "item121", "item111"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -217,7 +217,7 @@ func TestListItemsPaginated(t *testing.T) { // starred, oldest first starred := STARRED - have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false)) + have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false)) want = []string{"item212", "item013"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -233,7 +233,7 @@ func TestMarkItemsRead(t *testing.T) { db1 := testDB() testItemsSetup(db1) db1.MarkItemsRead(MarkFilter{}) - have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false)) + have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false)) want := []string{ "item111", "item112", "item121", "item122", "item211", "item011", "item012", @@ -247,7 +247,7 @@ func TestMarkItemsRead(t *testing.T) { db2 := testDB() scope2 := testItemsSetup(db2) db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id}) - have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false)) + have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false)) want = []string{ "item111", "item112", "item121", "item122", "item211", "item012", @@ -261,7 +261,7 @@ func TestMarkItemsRead(t *testing.T) { db3 := testDB() scope3 := testItemsSetup(db3) db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id}) - have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false)) + have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false)) want = []string{ "item111", "item112", "item122", "item211", "item012", @@ -319,7 +319,7 @@ func TestDeleteOldItems(t *testing.T) { } db.DeleteOldItems() - feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false) + feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false, false) if len(feedItems) != len(items)-3 { t.Fatalf( "invalid number of old items kept\nwant: %d\nhave: %d",