From aef569c33dea6c43ad6e71bd3abe5fff6beb93c4 Mon Sep 17 00:00:00 2001 From: icefed Date: Tue, 29 Nov 2022 21:06:07 +0800 Subject: [PATCH 1/4] fever api support Fever API spec: https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md --- src/server/auth/middleware.go | 16 +- src/server/fever.go | 340 ++++++++++++++++++++++++++++++++++ src/server/routes.go | 9 +- src/storage/item.go | 30 ++- 4 files changed, 382 insertions(+), 13 deletions(-) create mode 100644 src/server/fever.go diff --git a/src/server/auth/middleware.go b/src/server/auth/middleware.go index 4aca787..e6c7a0f 100644 --- a/src/server/auth/middleware.go +++ b/src/server/auth/middleware.go @@ -9,10 +9,10 @@ import ( ) type Middleware struct { - Username string - Password string - BasePath string - Public string + Username string + Password string + BasePath string + SkipAuthPaths []string } func unsafeMethod(method string) bool { @@ -20,9 +20,11 @@ func unsafeMethod(method string) bool { } func (m *Middleware) Handler(c *router.Context) { - if strings.HasPrefix(c.Req.URL.Path, m.BasePath+m.Public) { - c.Next() - return + for _, path := range m.SkipAuthPaths { + if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) { + c.Next() + return + } } if IsAuthenticated(c.Req, m.Username, m.Password) { c.Next() diff --git a/src/server/fever.go b/src/server/fever.go new file mode 100644 index 0000000..9fe788c --- /dev/null +++ b/src/server/fever.go @@ -0,0 +1,340 @@ +package server + +import ( + "crypto/md5" + "encoding/base64" + "fmt" + "log" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/nkanaev/yarr/src/server/router" + "github.com/nkanaev/yarr/src/storage" +) + +type FeverGroup struct { + ID int64 `json:"id"` + Title string `json:"title"` +} + +type FeverFeedsGroup struct { + GroupID int64 `json:"group_id"` + FeedIDs string `json:"feed_ids"` +} + +type FeverFeed struct { + ID int64 `json:"id"` + FaviconID int64 `json:"favicon_id"` + Title string `json:"title"` + Url string `json:"url"` + SiteUrl string `json:"site_url"` + IsSpark int `json:"is_spark"` + LastUpdated int64 `json:"last_updated_on_time"` +} + +type FeverItem struct { + ID int64 `json:"id"` + FeedID int64 `json:"feed_id"` + Title string `json:"title"` + Author string `json:"author"` + HTML string `json:"html"` + Url string `json:"url"` + IsSaved int `json:"is_saved"` + IsRead int `json:"is_read"` + CreatedAt int64 `json:"created_on_time"` +} + +type FeverFavicon struct { + ID int64 `json:"id"` + Data string `json:"data"` +} + +func writeFeverJSON(c *router.Context, data map[string]interface{}) { + data["api_version"] = 1 + data["auth"] = 1 + data["last_refreshed_on_time"] = time.Now().Unix() + c.JSON(http.StatusOK, data) +} + +func (s *Server) feverAuth(c *router.Context) bool { + if s.Username != "" && s.Password != "" { + apiKey := c.Req.FormValue("api_key") + md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password))) + if apiKey != fmt.Sprintf("%x", md5HashValue[:]) { + return false + } + } + return true +} + +func formHasValue(values url.Values, value string) bool { + if _, ok := values[value]; ok { + return true + } + return false +} + +func (s *Server) handleFever(c *router.Context) { + c.Req.ParseForm() + if !s.feverAuth(c) { + c.JSON(http.StatusOK, map[string]interface{}{ + "api_version": 1, + "auth": 0, + "last_refreshed_on_time": 0, + }) + return + } + + switch { + case formHasValue(c.Req.Form, "groups"): + s.feverGroupsHandler(c) + case formHasValue(c.Req.Form, "feeds"): + s.feverFeedsHandler(c) + case formHasValue(c.Req.Form, "unread_item_ids"): + s.feverUnreadItemIDsHandler(c) + case formHasValue(c.Req.Form, "saved_item_ids"): + s.feverSavedItemIDsHandler(c) + case formHasValue(c.Req.Form, "favicons"): + s.feverFaviconsHandler(c) + case formHasValue(c.Req.Form, "items"): + s.feverItemsHandler(c) + case formHasValue(c.Req.Form, "links"): + s.feverLinksHandler(c) + case formHasValue(c.Req.Form, "mark"): + s.feverMarkHandler(c) + default: + c.JSON(http.StatusOK, map[string]interface{}{ + "api_version": 1, + "auth": 1, + "last_refreshed_on_time": 0, + }) + } +} + +func joinInts(values []int64) string { + var result strings.Builder + for i, val := range values { + fmt.Fprintf(&result, "%d", val) + if i != len(values)-1 { + result.WriteString(",") + } + } + return result.String() +} + +func feedGroups(db *storage.Storage) []*FeverFeedsGroup { + feeds := db.ListFeeds() + + groupFeeds := make(map[int64][]int64) + for _, feed := range feeds { + // TODO: what about top-level feeds? + if feed.FolderId == nil { + continue + } + groupFeeds[*feed.FolderId] = append(groupFeeds[*feed.FolderId], feed.Id) + } + result := make([]*FeverFeedsGroup, 0) + for groupId, feedIds := range groupFeeds { + result = append(result, &FeverFeedsGroup{ + GroupID: groupId, + FeedIDs: joinInts(feedIds), + }) + } + return result +} + +func (s *Server) feverGroupsHandler(c *router.Context) { + folders := s.db.ListFolders() + groups := make([]*FeverGroup, len(folders)) + for i, folder := range folders { + groups[i] = &FeverGroup{ID: folder.Id, Title: folder.Title} + } + writeFeverJSON(c, map[string]interface{}{ + "groups": groups, + "feeds_groups": feedGroups(s.db), + }) +} + +func (s *Server) feverFeedsHandler(c *router.Context) { + feeds := s.db.ListFeeds() + httpStates := s.db.ListHTTPStates() + + feverFeeds := make([]*FeverFeed, len(feeds)) + for i, feed := range feeds { + var lastUpdated int64 + if state, ok := httpStates[feed.Id]; ok { + lastUpdated = state.LastRefreshed.Unix() + } + feverFeeds[i] = &FeverFeed{ + ID: feed.Id, + FaviconID: feed.Id, + Title: feed.Title, + Url: feed.FeedLink, + SiteUrl: feed.Link, + IsSpark: 0, + LastUpdated: lastUpdated, + } + } + writeFeverJSON(c, map[string]interface{}{ + "feeds": feverFeeds, + "feeds_groups": feedGroups(s.db), + }) +} + +func (s *Server) feverUnreadItemIDsHandler(c *router.Context) { + status := storage.UNREAD + itemIds := make([]int64, 0, 1000) + batch := 10000 + + items := s.db.ListItems(storage.ItemFilter{ + Status: &status, + }, batch, true) + for _, item := range items { + itemIds = append(itemIds, item.Id) + } + writeFeverJSON(c, map[string]interface{}{ + "unread_item_ids": joinInts(itemIds), + }) +} + +func (s *Server) feverSavedItemIDsHandler(c *router.Context) { + status := storage.STARRED + itemIds := make([]int64, 0, 1000) + batch := 10000 + + items := s.db.ListItems(storage.ItemFilter{ + Status: &status, + }, batch, true) + for _, item := range items { + itemIds = append(itemIds, item.Id) + } + writeFeverJSON(c, map[string]interface{}{ + "saved_item_ids": joinInts(itemIds), + }) +} + +func (s *Server) feverFaviconsHandler(c *router.Context) { + feeds := s.db.ListFeeds() + favicons := make([]*FeverFavicon, len(feeds)) + for i, feed := range feeds { + data := "data:image/gif;base64,R0lGODlhAQABAAAAACw=" + if feed.HasIcon { + icon := s.db.GetFeed(feed.Id).Icon + data = fmt.Sprintf( + "data:%s;base64,%s", + http.DetectContentType(*icon), + base64.StdEncoding.EncodeToString(*icon), + ) + } + favicons[i] = &FeverFavicon{ID: feed.Id, Data: data} + } + + writeFeverJSON(c, map[string]interface{}{ + "favicons": favicons, + }) +} + +func (s *Server) feverItemsHandler(c *router.Context) { + filter := storage.ItemFilter{} + query := c.Req.URL.Query() + + switch { + case query.Get("with_ids") != "": + ids := make([]int64, 0) + for _, idstr := range strings.Split(query.Get("with_ids"), ",") { + if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil { + ids = append(ids, idnum) + } + } + filter.IDs = &ids + case query.Get("since_id") != "": + idstr := query.Get("since_id") + if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil { + filter.SinceID = &idnum + } + case query.Get("max_id") != "": + idstr := query.Get("max_id") + if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil { + filter.MaxID = &idnum + } + } + + items := s.db.ListItems(filter, 50, true) + + feverItems := make([]FeverItem, len(items)) + for i, item := range items { + date := item.Date + time := date.Unix() + + isSaved := 0 + if item.Status == storage.STARRED { + isSaved = 1 + } + isRead := 0 + if item.Status == storage.READ { + isRead = 1 + } + feverItems[i] = FeverItem{ + ID: item.Id, + FeedID: item.FeedId, + Title: item.Title, + Author: "", + HTML: item.Content, + Url: item.Link, + IsSaved: isSaved, + IsRead: isRead, + CreatedAt: time, + } + } + + writeFeverJSON(c, map[string]interface{}{ + "items": feverItems, + }) +} + +func (s *Server) feverLinksHandler(c *router.Context) { + writeFeverJSON(c, map[string]interface{}{ + "links": make([]interface{}, 0), + }) +} + +func (s *Server) feverMarkHandler(c *router.Context) { + id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64) + if err != nil { + log.Print("invalid id:", err) + return + } + + switch c.Req.Form.Get("mark") { + case "item": + var status storage.ItemStatus + switch c.Req.Form.Get("as") { + case "read": + status = storage.READ + case "unread": + status = storage.UNREAD + case "saved": + status = storage.STARRED + case "unsaved": + status = storage.READ + default: + c.Out.WriteHeader(http.StatusBadRequest) + return + } + s.db.UpdateItemStatus(id, status) + case "feed": + x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) + before := time.Unix(x, 0) + s.db.MarkItemsRead(storage.MarkFilter{FeedID: &id, Before: &before}) + case "group": + x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) + before := time.Unix(x, 0) + s.db.MarkItemsRead(storage.MarkFilter{FolderID: &id, Before: &before}) + default: + c.Out.WriteHeader(http.StatusBadRequest) + return + } +} diff --git a/src/server/routes.go b/src/server/routes.go index 1ab18a4..9bf1ee7 100644 --- a/src/server/routes.go +++ b/src/server/routes.go @@ -30,10 +30,10 @@ func (s *Server) handler() http.Handler { if s.Username != "" && s.Password != "" { a := &auth.Middleware{ - BasePath: s.BasePath, - Username: s.Username, - Password: s.Password, - Public: "/static", + BasePath: s.BasePath, + Username: s.Username, + Password: s.Password, + SkipAuthPaths: []string{"/static", "/fever"}, } r.Use(a.Handler) } @@ -56,6 +56,7 @@ func (s *Server) handler() http.Handler { r.For("/opml/export", s.handleOPMLExport) r.For("/page", s.handlePageCrawl) r.For("/logout", s.handleLogout) + r.For("/fever/", s.handleFever) return r } diff --git a/src/storage/item.go b/src/storage/item.go index 49772fe..85508a1 100644 --- a/src/storage/item.go +++ b/src/storage/item.go @@ -62,11 +62,16 @@ type ItemFilter struct { Status *ItemStatus Search *string After *int64 + IDs *[]int64 + SinceID *int64 + MaxID *int64 } type MarkFilter struct { FolderID *int64 FeedID *int64 + + Before *time.Time } func (s *Storage) CreateItems(items []Item) bool { @@ -140,6 +145,24 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = ?)", compare)) args = append(args, *filter.After) } + if filter.IDs != nil && len(*filter.IDs) > 0 { + qmarks := make([]string, len(*filter.IDs)) + idargs := make([]interface{}, len(*filter.IDs)) + for i, id := range *filter.IDs { + qmarks[i] = "?" + idargs[i] = id + } + cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")") + args = append(args, idargs...) + } + if filter.SinceID != nil { + cond = append(cond, "i.id > ?") + args = append(args, filter.SinceID) + } + if filter.MaxID != nil { + cond = append(cond, "i.id < ?") + args = append(args, filter.MaxID) + } predicate := "1" if len(cond) > 0 { @@ -157,11 +180,14 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It if !newestFirst { order = "date asc, id asc" } + if filter.IDs != nil || filter.SinceID != nil || filter.MaxID != nil { + order = "i.id asc" + } query := fmt.Sprintf(` select i.id, i.guid, i.feed_id, - i.title, i.link, i.date, + i.title, i.link, i.content, i.date, i.status, i.image, i.podcast_url from items i where %s @@ -177,7 +203,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It var x Item err = rows.Scan( &x.Id, &x.GUID, &x.FeedId, - &x.Title, &x.Link, &x.Date, + &x.Title, &x.Link, &x.Content, &x.Date, &x.Status, &x.ImageURL, &x.AudioURL, ) if err != nil { From 5f78128dc659d607ced228ff692956613a738606 Mon Sep 17 00:00:00 2001 From: icefed Date: Mon, 6 Feb 2023 20:30:20 +0800 Subject: [PATCH 2/4] fix review --- src/server/auth/middleware.go | 10 ++-- src/server/fever.go | 104 +++++++++++++++++++++------------- src/server/routes.go | 10 ++-- src/storage/item.go | 20 ++++--- src/storage/item_test.go | 32 +++++------ 5 files changed, 103 insertions(+), 73 deletions(-) diff --git a/src/server/auth/middleware.go b/src/server/auth/middleware.go index e6c7a0f..b559be9 100644 --- a/src/server/auth/middleware.go +++ b/src/server/auth/middleware.go @@ -9,10 +9,10 @@ import ( ) type Middleware struct { - Username string - Password string - BasePath string - SkipAuthPaths []string + Username string + Password string + BasePath string + Public []string } func unsafeMethod(method string) bool { @@ -20,7 +20,7 @@ func unsafeMethod(method string) bool { } func (m *Middleware) Handler(c *router.Context) { - for _, path := range m.SkipAuthPaths { + for _, path := range m.Public { if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) { c.Next() return diff --git a/src/server/fever.go b/src/server/fever.go index 9fe788c..712f2eb 100644 --- a/src/server/fever.go +++ b/src/server/fever.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/nkanaev/yarr/src/server/auth" "github.com/nkanaev/yarr/src/server/router" "github.com/nkanaev/yarr/src/storage" ) @@ -63,7 +64,8 @@ func (s *Server) feverAuth(c *router.Context) bool { if s.Username != "" && s.Password != "" { apiKey := c.Req.FormValue("api_key") md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password))) - if apiKey != fmt.Sprintf("%x", md5HashValue[:]) { + hexMD5HashValue := fmt.Sprintf("%x", md5HashValue[:]) + if auth.StringsEqual(apiKey, hexMD5HashValue) { return false } } @@ -130,7 +132,6 @@ func feedGroups(db *storage.Storage) []*FeverFeedsGroup { groupFeeds := make(map[int64][]int64) for _, feed := range feeds { - // TODO: what about top-level feeds? if feed.FolderId == nil { continue } @@ -184,38 +185,6 @@ func (s *Server) feverFeedsHandler(c *router.Context) { }) } -func (s *Server) feverUnreadItemIDsHandler(c *router.Context) { - status := storage.UNREAD - itemIds := make([]int64, 0, 1000) - batch := 10000 - - items := s.db.ListItems(storage.ItemFilter{ - Status: &status, - }, batch, true) - for _, item := range items { - itemIds = append(itemIds, item.Id) - } - writeFeverJSON(c, map[string]interface{}{ - "unread_item_ids": joinInts(itemIds), - }) -} - -func (s *Server) feverSavedItemIDsHandler(c *router.Context) { - status := storage.STARRED - itemIds := make([]int64, 0, 1000) - batch := 10000 - - items := s.db.ListItems(storage.ItemFilter{ - Status: &status, - }, batch, true) - for _, item := range items { - itemIds = append(itemIds, item.Id) - } - writeFeverJSON(c, map[string]interface{}{ - "saved_item_ids": joinInts(itemIds), - }) -} - func (s *Server) feverFaviconsHandler(c *router.Context) { feeds := s.db.ListFeeds() favicons := make([]*FeverFavicon, len(feeds)) @@ -237,6 +206,10 @@ func (s *Server) feverFaviconsHandler(c *router.Context) { }) } +// for memory pressure reasons, we only return a limited number of items +// documented at https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md#items +const listLimit = 50 + func (s *Server) feverItemsHandler(c *router.Context) { filter := storage.ItemFilter{} query := c.Req.URL.Query() @@ -262,7 +235,7 @@ func (s *Server) feverItemsHandler(c *router.Context) { } } - items := s.db.ListItems(filter, 50, true) + items := s.db.ListItems(filter, listLimit, true, true) feverItems := make([]FeverItem, len(items)) for i, item := range items { @@ -301,6 +274,50 @@ func (s *Server) feverLinksHandler(c *router.Context) { }) } +func (s *Server) feverUnreadItemIDsHandler(c *router.Context) { + status := storage.UNREAD + itemIds := make([]int64, 0) + + itemFilter := storage.ItemFilter{ + Status: &status, + } + for { + items := s.db.ListItems(itemFilter, listLimit, true, false) + if len(items) == 0 { + break + } + for _, item := range items { + itemIds = append(itemIds, item.Id) + } + itemFilter.After = &items[len(items)-1].Id + } + writeFeverJSON(c, map[string]interface{}{ + "unread_item_ids": joinInts(itemIds), + }) +} + +func (s *Server) feverSavedItemIDsHandler(c *router.Context) { + status := storage.STARRED + itemIds := make([]int64, 0) + + itemFilter := storage.ItemFilter{ + Status: &status, + } + for { + items := s.db.ListItems(itemFilter, listLimit, true, false) + if len(items) == 0 { + break + } + for _, item := range items { + itemIds = append(itemIds, item.Id) + } + itemFilter.After = &items[len(items)-1].Id + } + writeFeverJSON(c, map[string]interface{}{ + "saved_item_ids": joinInts(itemIds), + }) +} + func (s *Server) feverMarkHandler(c *router.Context) { id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64) if err != nil { @@ -326,13 +343,22 @@ func (s *Server) feverMarkHandler(c *router.Context) { } s.db.UpdateItemStatus(id, status) case "feed": + markFilter := storage.MarkFilter{FeedID: &id} x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) - before := time.Unix(x, 0) - s.db.MarkItemsRead(storage.MarkFilter{FeedID: &id, Before: &before}) + if x > 0 { + before := time.Unix(x, 0) + markFilter.Before = &before + } + s.db.MarkItemsRead(markFilter) + // s.db.MarkItemsRead(markFilter) case "group": + markFilter := storage.MarkFilter{FolderID: &id} x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) - before := time.Unix(x, 0) - s.db.MarkItemsRead(storage.MarkFilter{FolderID: &id, Before: &before}) + if x > 0 { + before := time.Unix(x, 0) + markFilter.Before = &before + } + s.db.MarkItemsRead(markFilter) default: c.Out.WriteHeader(http.StatusBadRequest) return diff --git a/src/server/routes.go b/src/server/routes.go index 9bf1ee7..7ddab31 100644 --- a/src/server/routes.go +++ b/src/server/routes.go @@ -30,10 +30,10 @@ func (s *Server) handler() http.Handler { if s.Username != "" && s.Password != "" { a := &auth.Middleware{ - BasePath: s.BasePath, - Username: s.Username, - Password: s.Password, - SkipAuthPaths: []string{"/static", "/fever"}, + BasePath: s.BasePath, + Username: s.Username, + Password: s.Password, + Public: []string{"/static", "/fever"}, } r.Use(a.Handler) } @@ -354,7 +354,7 @@ func (s *Server) handleItemList(c *router.Context) { } newestFirst := query.Get("oldest_first") != "true" - items := s.db.ListItems(filter, perPage+1, newestFirst) + items := s.db.ListItems(filter, perPage+1, newestFirst, false) hasMore := false if len(items) == perPage+1 { hasMore = true diff --git a/src/storage/item.go b/src/storage/item.go index 85508a1..a411be6 100644 --- a/src/storage/item.go +++ b/src/storage/item.go @@ -65,6 +65,7 @@ type ItemFilter struct { IDs *[]int64 SinceID *int64 MaxID *int64 + Before *time.Time } type MarkFilter struct { @@ -172,7 +173,7 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac return predicate, args } -func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []Item { +func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item { predicate, args := listQueryPredicate(filter, newestFirst) result := make([]Item, 0, 0) @@ -184,16 +185,19 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It order = "i.id asc" } + selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url" + if withContent { + selectCols += ", i.content" + } else { + selectCols += ", '' as content" + } query := fmt.Sprintf(` - select - i.id, i.guid, i.feed_id, - i.title, i.link, i.content, i.date, - i.status, i.image, i.podcast_url + select %s from items i where %s order by %s limit %d - `, predicate, order, limit) + `, selectCols, predicate, order, limit) rows, err := s.db.Query(query, args...) if err != nil { log.Print(err) @@ -203,8 +207,8 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It var x Item err = rows.Scan( &x.Id, &x.GUID, &x.FeedId, - &x.Title, &x.Link, &x.Content, &x.Date, - &x.Status, &x.ImageURL, &x.AudioURL, + &x.Title, &x.Link, &x.Date, + &x.Status, &x.ImageURL, &x.AudioURL, &x.Content, ) if err != nil { log.Print(err) diff --git a/src/storage/item_test.go b/src/storage/item_test.go index cfe5975..30b2885 100644 --- a/src/storage/item_test.go +++ b/src/storage/item_test.go @@ -104,7 +104,7 @@ func TestListItems(t *testing.T) { // filter by folder_id - have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false)) + have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false)) want := []string{"item111", "item112", "item113", "item121", "item122"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -112,7 +112,7 @@ func TestListItems(t *testing.T) { t.Fail() } - have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false)) want = []string{"item211", "item212"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -122,7 +122,7 @@ func TestListItems(t *testing.T) { // filter by feed_id - have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false)) want = []string{"item111", "item112", "item113"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -130,7 +130,7 @@ func TestListItems(t *testing.T) { t.Fail() } - have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false)) want = []string{"item011", "item012", "item013"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -141,7 +141,7 @@ func TestListItems(t *testing.T) { // filter by status var starred ItemStatus = STARRED - have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false)) want = []string{"item113", "item212", "item013"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -150,7 +150,7 @@ func TestListItems(t *testing.T) { } var unread ItemStatus = UNREAD - have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false)) + have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false)) want = []string{"item111", "item121", "item011"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -160,7 +160,7 @@ func TestListItems(t *testing.T) { // limit - have = getItemGuids(db.ListItems(ItemFilter{}, 2, false)) + have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false)) want = []string{"item111", "item112"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -171,7 +171,7 @@ func TestListItems(t *testing.T) { // filter by search db.SyncSearch() search1 := "title111" - have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true)) + have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false)) want = []string{"item111"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -180,7 +180,7 @@ func TestListItems(t *testing.T) { } // sort by date - have = getItemGuids(db.ListItems(ItemFilter{}, 4, true)) + have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false)) want = []string{"item013", "item012", "item011", "item212"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -197,7 +197,7 @@ func TestListItemsPaginated(t *testing.T) { item121 := getItem(db, "item121") // all, newest first - have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true)) + have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false)) want := []string{"item011", "item212", "item211"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -207,7 +207,7 @@ func TestListItemsPaginated(t *testing.T) { // unread, newest first unread := UNREAD - have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true)) + have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false)) want = []string{"item011", "item121", "item111"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -217,7 +217,7 @@ func TestListItemsPaginated(t *testing.T) { // starred, oldest first starred := STARRED - have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false)) + have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false)) want = []string{"item212", "item013"} if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -233,7 +233,7 @@ func TestMarkItemsRead(t *testing.T) { db1 := testDB() testItemsSetup(db1) db1.MarkItemsRead(MarkFilter{}) - have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false)) + have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false)) want := []string{ "item111", "item112", "item121", "item122", "item211", "item011", "item012", @@ -247,7 +247,7 @@ func TestMarkItemsRead(t *testing.T) { db2 := testDB() scope2 := testItemsSetup(db2) db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id}) - have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false)) + have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false)) want = []string{ "item111", "item112", "item121", "item122", "item211", "item012", @@ -261,7 +261,7 @@ func TestMarkItemsRead(t *testing.T) { db3 := testDB() scope3 := testItemsSetup(db3) db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id}) - have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false)) + have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false)) want = []string{ "item111", "item112", "item122", "item211", "item012", @@ -319,7 +319,7 @@ func TestDeleteOldItems(t *testing.T) { } db.DeleteOldItems() - feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false) + feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false, false) if len(feedItems) != len(items)-3 { t.Fatalf( "invalid number of old items kept\nwant: %d\nhave: %d", From 2db6621bff5d83e1f9710b816087ab00887798c6 Mon Sep 17 00:00:00 2001 From: icefed Date: Tue, 7 Feb 2023 21:40:51 +0800 Subject: [PATCH 3/4] update --- src/server/fever.go | 39 +++++++++++++++++++++++++++++---------- src/storage/item.go | 10 +++++++++- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/src/server/fever.go b/src/server/fever.go index 712f2eb..0ded40b 100644 --- a/src/server/fever.go +++ b/src/server/fever.go @@ -53,13 +53,27 @@ type FeverFavicon struct { Data string `json:"data"` } -func writeFeverJSON(c *router.Context, data map[string]interface{}) { +func writeFeverJSON(c *router.Context, data map[string]interface{}, lastRefreshed int64) { data["api_version"] = 1 data["auth"] = 1 - data["last_refreshed_on_time"] = time.Now().Unix() + data["last_refreshed_on_time"] = lastRefreshed c.JSON(http.StatusOK, data) } +func getLastRefreshedOnTime(httpStates map[int64]storage.HTTPState) int64 { + if len(httpStates) == 0 { + return 0 + } + + var lastRefreshed int64 + for _, state := range httpStates { + if state.LastRefreshed.Unix() > lastRefreshed { + lastRefreshed = state.LastRefreshed.Unix() + } + } + return lastRefreshed +} + func (s *Server) feverAuth(c *router.Context) bool { if s.Username != "" && s.Password != "" { apiKey := c.Req.FormValue("api_key") @@ -156,7 +170,7 @@ func (s *Server) feverGroupsHandler(c *router.Context) { writeFeverJSON(c, map[string]interface{}{ "groups": groups, "feeds_groups": feedGroups(s.db), - }) + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) } func (s *Server) feverFeedsHandler(c *router.Context) { @@ -182,7 +196,7 @@ func (s *Server) feverFeedsHandler(c *router.Context) { writeFeverJSON(c, map[string]interface{}{ "feeds": feverFeeds, "feeds_groups": feedGroups(s.db), - }) + }, getLastRefreshedOnTime(httpStates)) } func (s *Server) feverFaviconsHandler(c *router.Context) { @@ -203,7 +217,7 @@ func (s *Server) feverFaviconsHandler(c *router.Context) { writeFeverJSON(c, map[string]interface{}{ "favicons": favicons, - }) + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) } // for memory pressure reasons, we only return a limited number of items @@ -265,13 +279,13 @@ func (s *Server) feverItemsHandler(c *router.Context) { writeFeverJSON(c, map[string]interface{}{ "items": feverItems, - }) + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) } func (s *Server) feverLinksHandler(c *router.Context) { writeFeverJSON(c, map[string]interface{}{ "links": make([]interface{}, 0), - }) + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) } func (s *Server) feverUnreadItemIDsHandler(c *router.Context) { @@ -293,7 +307,7 @@ func (s *Server) feverUnreadItemIDsHandler(c *router.Context) { } writeFeverJSON(c, map[string]interface{}{ "unread_item_ids": joinInts(itemIds), - }) + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) } func (s *Server) feverSavedItemIDsHandler(c *router.Context) { @@ -315,7 +329,7 @@ func (s *Server) feverSavedItemIDsHandler(c *router.Context) { } writeFeverJSON(c, map[string]interface{}{ "saved_item_ids": joinInts(itemIds), - }) + }, getLastRefreshedOnTime(s.db.ListHTTPStates())) } func (s *Server) feverMarkHandler(c *router.Context) { @@ -343,6 +357,9 @@ func (s *Server) feverMarkHandler(c *router.Context) { } s.db.UpdateItemStatus(id, status) case "feed": + if c.Req.Form.Get("as") != "read" { + c.Out.WriteHeader(http.StatusBadRequest) + } markFilter := storage.MarkFilter{FeedID: &id} x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) if x > 0 { @@ -350,8 +367,10 @@ func (s *Server) feverMarkHandler(c *router.Context) { markFilter.Before = &before } s.db.MarkItemsRead(markFilter) - // s.db.MarkItemsRead(markFilter) case "group": + if c.Req.Form.Get("as") != "read" { + c.Out.WriteHeader(http.StatusBadRequest) + } markFilter := storage.MarkFilter{FolderID: &id} x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64) if x > 0 { diff --git a/src/storage/item.go b/src/storage/item.go index a411be6..6b63343 100644 --- a/src/storage/item.go +++ b/src/storage/item.go @@ -164,6 +164,10 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac cond = append(cond, "i.id < ?") args = append(args, filter.MaxID) } + if filter.Before != nil { + cond = append(cond, "i.date < ?") + args = append(args, filter.Before) + } predicate := "1" if len(cond) > 0 { @@ -244,7 +248,11 @@ func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool { } func (s *Storage) MarkItemsRead(filter MarkFilter) bool { - predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID}, false) + predicate, args := listQueryPredicate(ItemFilter{ + FolderID: filter.FolderID, + FeedID: filter.FeedID, + Before: filter.Before, + }, false) query := fmt.Sprintf(` update items as i set status = %d where %s and i.status != %d From ee34959d7c7c5a7e689026f1825e14e661e5a034 Mon Sep 17 00:00:00 2001 From: icefed Date: Tue, 7 Feb 2023 21:44:25 +0800 Subject: [PATCH 4/4] update --- src/server/fever.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/fever.go b/src/server/fever.go index 0ded40b..0f17a1a 100644 --- a/src/server/fever.go +++ b/src/server/fever.go @@ -125,7 +125,7 @@ func (s *Server) handleFever(c *router.Context) { c.JSON(http.StatusOK, map[string]interface{}{ "api_version": 1, "auth": 1, - "last_refreshed_on_time": 0, + "last_refreshed_on_time": getLastRefreshedOnTime(s.db.ListHTTPStates()), }) } }