fix review

src/server/auth/middleware.go

@@ -12,7 +12,7 @@ type Middleware struct {
 	Username string
 	Password string
 	BasePath string
-	SkipAuthPaths []string
+	Public   []string
 }
 
 func unsafeMethod(method string) bool {
@@ -20,7 +20,7 @@ func unsafeMethod(method string) bool {
 }
 
 func (m *Middleware) Handler(c *router.Context) {
-	for _, path := range m.SkipAuthPaths {
+	for _, path := range m.Public {
 		if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) {
 			c.Next()
 			return

src/server/fever.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/nkanaev/yarr/src/server/auth"
 	"github.com/nkanaev/yarr/src/server/router"
 	"github.com/nkanaev/yarr/src/storage"
 )
@@ -63,7 +64,8 @@ func (s *Server) feverAuth(c *router.Context) bool {
 	if s.Username != "" && s.Password != "" {
 		apiKey := c.Req.FormValue("api_key")
 		md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password)))
-		if apiKey != fmt.Sprintf("%x", md5HashValue[:]) {
+		hexMD5HashValue := fmt.Sprintf("%x", md5HashValue[:])
+		if auth.StringsEqual(apiKey, hexMD5HashValue) {
 			return false
 		}
 	}
@@ -130,7 +132,6 @@ func feedGroups(db *storage.Storage) []*FeverFeedsGroup {
 
 	groupFeeds := make(map[int64][]int64)
 	for _, feed := range feeds {
-		// TODO: what about top-level feeds?
 		if feed.FolderId == nil {
 			continue
 		}
@@ -184,38 +185,6 @@ func (s *Server) feverFeedsHandler(c *router.Context) {
 	})
 }
 
-func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
-	status := storage.UNREAD
-	itemIds := make([]int64, 0, 1000)
-	batch := 10000
-
-	items := s.db.ListItems(storage.ItemFilter{
-		Status: &status,
-	}, batch, true)
-	for _, item := range items {
-		itemIds = append(itemIds, item.Id)
-	}
-	writeFeverJSON(c, map[string]interface{}{
-		"unread_item_ids": joinInts(itemIds),
-	})
-}
-
-func (s *Server) feverSavedItemIDsHandler(c *router.Context) {
-	status := storage.STARRED
-	itemIds := make([]int64, 0, 1000)
-	batch := 10000
-
-	items := s.db.ListItems(storage.ItemFilter{
-		Status: &status,
-	}, batch, true)
-	for _, item := range items {
-		itemIds = append(itemIds, item.Id)
-	}
-	writeFeverJSON(c, map[string]interface{}{
-		"saved_item_ids": joinInts(itemIds),
-	})
-}
-
 func (s *Server) feverFaviconsHandler(c *router.Context) {
 	feeds := s.db.ListFeeds()
 	favicons := make([]*FeverFavicon, len(feeds))
@@ -237,6 +206,10 @@ func (s *Server) feverFaviconsHandler(c *router.Context) {
 	})
 }
 
+// for memory pressure reasons, we only return a limited number of items
+// documented at https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md#items
+const listLimit = 50
+
 func (s *Server) feverItemsHandler(c *router.Context) {
 	filter := storage.ItemFilter{}
 	query := c.Req.URL.Query()
@@ -262,7 +235,7 @@ func (s *Server) feverItemsHandler(c *router.Context) {
 		}
 	}
 
-	items := s.db.ListItems(filter, 50, true)
+	items := s.db.ListItems(filter, listLimit, true, true)
 
 	feverItems := make([]FeverItem, len(items))
 	for i, item := range items {
@@ -301,6 +274,50 @@ func (s *Server) feverLinksHandler(c *router.Context) {
 	})
 }
 
+func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
+	status := storage.UNREAD
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"unread_item_ids": joinInts(itemIds),
+	})
+}
+
+func (s *Server) feverSavedItemIDsHandler(c *router.Context) {
+	status := storage.STARRED
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"saved_item_ids": joinInts(itemIds),
+	})
+}
+
 func (s *Server) feverMarkHandler(c *router.Context) {
 	id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64)
 	if err != nil {
@@ -326,13 +343,22 @@ func (s *Server) feverMarkHandler(c *router.Context) {
 		}
 		s.db.UpdateItemStatus(id, status)
 	case "feed":
+		markFilter := storage.MarkFilter{FeedID: &id}
 		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
 			before := time.Unix(x, 0)
-		s.db.MarkItemsRead(storage.MarkFilter{FeedID: &id, Before: &before})
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
+		// s.db.MarkItemsRead(markFilter)
 	case "group":
+		markFilter := storage.MarkFilter{FolderID: &id}
 		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
 			before := time.Unix(x, 0)
-		s.db.MarkItemsRead(storage.MarkFilter{FolderID: &id, Before: &before})
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
 	default:
 		c.Out.WriteHeader(http.StatusBadRequest)
 		return

src/server/routes.go

@@ -34,7 +34,7 @@ func (s *Server) handler() http.Handler {
 			BasePath: s.BasePath,
 			Username: s.Username,
 			Password: s.Password,
-			SkipAuthPaths: []string{"/static", "/fever"},
+			Public:   []string{"/static", "/fever"},
 		}
 		r.Use(a.Handler)
 	}
@@ -365,7 +365,7 @@ func (s *Server) handleItemList(c *router.Context) {
 		}
 		newestFirst := query.Get("oldest_first") != "true"
 
-		items := s.db.ListItems(filter, perPage+1, newestFirst)
+		items := s.db.ListItems(filter, perPage+1, newestFirst, false)
 		hasMore := false
 		if len(items) == perPage+1 {
 			hasMore = true

src/storage/item.go

@@ -65,6 +65,7 @@ type ItemFilter struct {
 	IDs      *[]int64
 	SinceID  *int64
 	MaxID    *int64
+	Before   *time.Time
 }
 
 type MarkFilter struct {
@@ -172,7 +173,7 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
 	return predicate, args
 }
 
-func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []Item {
+func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item {
 	predicate, args := listQueryPredicate(filter, newestFirst)
 	result := make([]Item, 0, 0)
 
@@ -184,16 +185,19 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
 		order = "i.id asc"
 	}
 
+	selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url"
+	if withContent {
+		selectCols += ", i.content"
+	} else {
+		selectCols += ", '' as content"
+	}
 	query := fmt.Sprintf(`
-		select
+		select %s
-			i.id, i.guid, i.feed_id,
-			i.title, i.link, i.content, i.date,
-			i.status, i.image, i.podcast_url
 		from items i
 		where %s
 		order by %s
 		limit %d
-		`, predicate, order, limit)
+		`, selectCols, predicate, order, limit)
 	rows, err := s.db.Query(query, args...)
 	if err != nil {
 		log.Print(err)
@@ -203,8 +207,8 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
 		var x Item
 		err = rows.Scan(
 			&x.Id, &x.GUID, &x.FeedId,
-			&x.Title, &x.Link, &x.Content, &x.Date,
+			&x.Title, &x.Link, &x.Date,
-			&x.Status, &x.ImageURL, &x.AudioURL,
+			&x.Status, &x.ImageURL, &x.AudioURL, &x.Content,
 		)
 		if err != nil {
 			log.Print(err)

src/storage/item_test.go

@@ -104,7 +104,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by folder_id
 
-	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false))
+	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
 	want := []string{"item111", "item112", "item113", "item121", "item122"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -112,7 +112,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
 	want = []string{"item211", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -122,7 +122,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by feed_id
 
-	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
 	want = []string{"item111", "item112", "item113"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -130,7 +130,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
 	want = []string{"item011", "item012", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -141,7 +141,7 @@ func TestListItems(t *testing.T) {
 	// filter by status
 
 	var starred ItemStatus = STARRED
-	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
 	want = []string{"item113", "item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -150,7 +150,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	var unread ItemStatus = UNREAD
-	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
 	want = []string{"item111", "item121", "item011"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -160,7 +160,7 @@ func TestListItems(t *testing.T) {
 
 	// limit
 
-	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
 	want = []string{"item111", "item112"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -171,7 +171,7 @@ func TestListItems(t *testing.T) {
 	// filter by search
 	db.SyncSearch()
 	search1 := "title111"
-	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
 	want = []string{"item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -180,7 +180,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	// sort by date
-	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
 	want = []string{"item013", "item012", "item011", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -197,7 +197,7 @@ func TestListItemsPaginated(t *testing.T) {
 	item121 := getItem(db, "item121")
 
 	// all, newest first
-	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true))
+	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
 	want := []string{"item011", "item212", "item211"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -207,7 +207,7 @@ func TestListItemsPaginated(t *testing.T) {
 
 	// unread, newest first
 	unread := UNREAD
-	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true))
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false))
 	want = []string{"item011", "item121", "item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -217,7 +217,7 @@ func TestListItemsPaginated(t *testing.T) {
 
 	// starred, oldest first
 	starred := STARRED
-	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false))
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false))
 	want = []string{"item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -233,7 +233,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db1 := testDB()
 	testItemsSetup(db1)
 	db1.MarkItemsRead(MarkFilter{})
-	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false))
+	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want := []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item011", "item012",
@@ -247,7 +247,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db2 := testDB()
 	scope2 := testItemsSetup(db2)
 	db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
-	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false))
+	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item012",
@@ -261,7 +261,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db3 := testDB()
 	scope3 := testItemsSetup(db3)
 	db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
-	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false))
+	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item122",
 		"item211", "item012",
@@ -319,7 +319,7 @@ func TestDeleteOldItems(t *testing.T) {
 	}
 
 	db.DeleteOldItems()
-	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false)
+	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false, false)
 	if len(feedItems) != len(items)-3 {
 		t.Fatalf(
 			"invalid number of old items kept\nwant: %d\nhave: %d",