Merge ee34959d7c7c5a7e689026f1825e14e661e5a034 into 479aebd0233daea2b46fd65c90a6134f133e3c15

Fever API spec: https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md

doc/changelog.txt

@@ -1,6 +1,7 @@
 # upcoming 
 
 - (fix) duplicate articles caused by the same feed addition (thanks to @adaszko)
+- (fix) relative article links (thanks to @adazsko for the report)
 
 # v2.4 (2023-08-15)
 

src/server/auth/middleware.go

@@ -12,7 +12,7 @@ type Middleware struct {
 	Username string
 	Password string
 	BasePath string
-	Public   string
+	Public   []string
 }
 
 func unsafeMethod(method string) bool {
@@ -20,9 +20,11 @@ func unsafeMethod(method string) bool {
 }
 
 func (m *Middleware) Handler(c *router.Context) {
-	if strings.HasPrefix(c.Req.URL.Path, m.BasePath+m.Public) {
-		c.Next()
-		return
+	for _, path := range m.Public {
+		if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) {
+			c.Next()
+			return
+		}
 	}
 	if IsAuthenticated(c.Req, m.Username, m.Password) {
 		c.Next()

src/server/fever.go

@@ -0,0 +1,385 @@
+package server
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/nkanaev/yarr/src/server/auth"
+	"github.com/nkanaev/yarr/src/server/router"
+	"github.com/nkanaev/yarr/src/storage"
+)
+
+type FeverGroup struct {
+	ID    int64  `json:"id"`
+	Title string `json:"title"`
+}
+
+type FeverFeedsGroup struct {
+	GroupID int64  `json:"group_id"`
+	FeedIDs string `json:"feed_ids"`
+}
+
+type FeverFeed struct {
+	ID          int64  `json:"id"`
+	FaviconID   int64  `json:"favicon_id"`
+	Title       string `json:"title"`
+	Url         string `json:"url"`
+	SiteUrl     string `json:"site_url"`
+	IsSpark     int    `json:"is_spark"`
+	LastUpdated int64  `json:"last_updated_on_time"`
+}
+
+type FeverItem struct {
+	ID        int64  `json:"id"`
+	FeedID    int64  `json:"feed_id"`
+	Title     string `json:"title"`
+	Author    string `json:"author"`
+	HTML      string `json:"html"`
+	Url       string `json:"url"`
+	IsSaved   int    `json:"is_saved"`
+	IsRead    int    `json:"is_read"`
+	CreatedAt int64  `json:"created_on_time"`
+}
+
+type FeverFavicon struct {
+	ID   int64  `json:"id"`
+	Data string `json:"data"`
+}
+
+func writeFeverJSON(c *router.Context, data map[string]interface{}, lastRefreshed int64) {
+	data["api_version"] = 1
+	data["auth"] = 1
+	data["last_refreshed_on_time"] = lastRefreshed
+	c.JSON(http.StatusOK, data)
+}
+
+func getLastRefreshedOnTime(httpStates map[int64]storage.HTTPState) int64 {
+	if len(httpStates) == 0 {
+		return 0
+	}
+
+	var lastRefreshed int64
+	for _, state := range httpStates {
+		if state.LastRefreshed.Unix() > lastRefreshed {
+			lastRefreshed = state.LastRefreshed.Unix()
+		}
+	}
+	return lastRefreshed
+}
+
+func (s *Server) feverAuth(c *router.Context) bool {
+	if s.Username != "" && s.Password != "" {
+		apiKey := c.Req.FormValue("api_key")
+		md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password)))
+		hexMD5HashValue := fmt.Sprintf("%x", md5HashValue[:])
+		if auth.StringsEqual(apiKey, hexMD5HashValue) {
+			return false
+		}
+	}
+	return true
+}
+
+func formHasValue(values url.Values, value string) bool {
+	if _, ok := values[value]; ok {
+		return true
+	}
+	return false
+}
+
+func (s *Server) handleFever(c *router.Context) {
+	c.Req.ParseForm()
+	if !s.feverAuth(c) {
+		c.JSON(http.StatusOK, map[string]interface{}{
+			"api_version":            1,
+			"auth":                   0,
+			"last_refreshed_on_time": 0,
+		})
+		return
+	}
+
+	switch {
+	case formHasValue(c.Req.Form, "groups"):
+		s.feverGroupsHandler(c)
+	case formHasValue(c.Req.Form, "feeds"):
+		s.feverFeedsHandler(c)
+	case formHasValue(c.Req.Form, "unread_item_ids"):
+		s.feverUnreadItemIDsHandler(c)
+	case formHasValue(c.Req.Form, "saved_item_ids"):
+		s.feverSavedItemIDsHandler(c)
+	case formHasValue(c.Req.Form, "favicons"):
+		s.feverFaviconsHandler(c)
+	case formHasValue(c.Req.Form, "items"):
+		s.feverItemsHandler(c)
+	case formHasValue(c.Req.Form, "links"):
+		s.feverLinksHandler(c)
+	case formHasValue(c.Req.Form, "mark"):
+		s.feverMarkHandler(c)
+	default:
+		c.JSON(http.StatusOK, map[string]interface{}{
+			"api_version":            1,
+			"auth":                   1,
+			"last_refreshed_on_time": getLastRefreshedOnTime(s.db.ListHTTPStates()),
+		})
+	}
+}
+
+func joinInts(values []int64) string {
+	var result strings.Builder
+	for i, val := range values {
+		fmt.Fprintf(&result, "%d", val)
+		if i != len(values)-1 {
+			result.WriteString(",")
+		}
+	}
+	return result.String()
+}
+
+func feedGroups(db *storage.Storage) []*FeverFeedsGroup {
+	feeds := db.ListFeeds()
+
+	groupFeeds := make(map[int64][]int64)
+	for _, feed := range feeds {
+		if feed.FolderId == nil {
+			continue
+		}
+		groupFeeds[*feed.FolderId] = append(groupFeeds[*feed.FolderId], feed.Id)
+	}
+	result := make([]*FeverFeedsGroup, 0)
+	for groupId, feedIds := range groupFeeds {
+		result = append(result, &FeverFeedsGroup{
+			GroupID: groupId,
+			FeedIDs: joinInts(feedIds),
+		})
+	}
+	return result
+}
+
+func (s *Server) feverGroupsHandler(c *router.Context) {
+	folders := s.db.ListFolders()
+	groups := make([]*FeverGroup, len(folders))
+	for i, folder := range folders {
+		groups[i] = &FeverGroup{ID: folder.Id, Title: folder.Title}
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"groups":       groups,
+		"feeds_groups": feedGroups(s.db),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverFeedsHandler(c *router.Context) {
+	feeds := s.db.ListFeeds()
+	httpStates := s.db.ListHTTPStates()
+
+	feverFeeds := make([]*FeverFeed, len(feeds))
+	for i, feed := range feeds {
+		var lastUpdated int64
+		if state, ok := httpStates[feed.Id]; ok {
+			lastUpdated = state.LastRefreshed.Unix()
+		}
+		feverFeeds[i] = &FeverFeed{
+			ID:          feed.Id,
+			FaviconID:   feed.Id,
+			Title:       feed.Title,
+			Url:         feed.FeedLink,
+			SiteUrl:     feed.Link,
+			IsSpark:     0,
+			LastUpdated: lastUpdated,
+		}
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"feeds":        feverFeeds,
+		"feeds_groups": feedGroups(s.db),
+	}, getLastRefreshedOnTime(httpStates))
+}
+
+func (s *Server) feverFaviconsHandler(c *router.Context) {
+	feeds := s.db.ListFeeds()
+	favicons := make([]*FeverFavicon, len(feeds))
+	for i, feed := range feeds {
+		data := "data:image/gif;base64,R0lGODlhAQABAAAAACw="
+		if feed.HasIcon {
+			icon := s.db.GetFeed(feed.Id).Icon
+			data = fmt.Sprintf(
+				"data:%s;base64,%s",
+				http.DetectContentType(*icon),
+				base64.StdEncoding.EncodeToString(*icon),
+			)
+		}
+		favicons[i] = &FeverFavicon{ID: feed.Id, Data: data}
+	}
+
+	writeFeverJSON(c, map[string]interface{}{
+		"favicons": favicons,
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+// for memory pressure reasons, we only return a limited number of items
+// documented at https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md#items
+const listLimit = 50
+
+func (s *Server) feverItemsHandler(c *router.Context) {
+	filter := storage.ItemFilter{}
+	query := c.Req.URL.Query()
+
+	switch {
+	case query.Get("with_ids") != "":
+		ids := make([]int64, 0)
+		for _, idstr := range strings.Split(query.Get("with_ids"), ",") {
+			if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+				ids = append(ids, idnum)
+			}
+		}
+		filter.IDs = &ids
+	case query.Get("since_id") != "":
+		idstr := query.Get("since_id")
+		if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+			filter.SinceID = &idnum
+		}
+	case query.Get("max_id") != "":
+		idstr := query.Get("max_id")
+		if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+			filter.MaxID = &idnum
+		}
+	}
+
+	items := s.db.ListItems(filter, listLimit, true, true)
+
+	feverItems := make([]FeverItem, len(items))
+	for i, item := range items {
+		date := item.Date
+		time := date.Unix()
+
+		isSaved := 0
+		if item.Status == storage.STARRED {
+			isSaved = 1
+		}
+		isRead := 0
+		if item.Status == storage.READ {
+			isRead = 1
+		}
+		feverItems[i] = FeverItem{
+			ID:        item.Id,
+			FeedID:    item.FeedId,
+			Title:     item.Title,
+			Author:    "",
+			HTML:      item.Content,
+			Url:       item.Link,
+			IsSaved:   isSaved,
+			IsRead:    isRead,
+			CreatedAt: time,
+		}
+	}
+
+	writeFeverJSON(c, map[string]interface{}{
+		"items": feverItems,
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverLinksHandler(c *router.Context) {
+	writeFeverJSON(c, map[string]interface{}{
+		"links": make([]interface{}, 0),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
+	status := storage.UNREAD
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"unread_item_ids": joinInts(itemIds),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverSavedItemIDsHandler(c *router.Context) {
+	status := storage.STARRED
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"saved_item_ids": joinInts(itemIds),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverMarkHandler(c *router.Context) {
+	id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64)
+	if err != nil {
+		log.Print("invalid id:", err)
+		return
+	}
+
+	switch c.Req.Form.Get("mark") {
+	case "item":
+		var status storage.ItemStatus
+		switch c.Req.Form.Get("as") {
+		case "read":
+			status = storage.READ
+		case "unread":
+			status = storage.UNREAD
+		case "saved":
+			status = storage.STARRED
+		case "unsaved":
+			status = storage.READ
+		default:
+			c.Out.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		s.db.UpdateItemStatus(id, status)
+	case "feed":
+		if c.Req.Form.Get("as") != "read" {
+			c.Out.WriteHeader(http.StatusBadRequest)
+		}
+		markFilter := storage.MarkFilter{FeedID: &id}
+		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
+			before := time.Unix(x, 0)
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
+	case "group":
+		if c.Req.Form.Get("as") != "read" {
+			c.Out.WriteHeader(http.StatusBadRequest)
+		}
+		markFilter := storage.MarkFilter{FolderID: &id}
+		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
+			before := time.Unix(x, 0)
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
+	default:
+		c.Out.WriteHeader(http.StatusBadRequest)
+		return
+	}
+}

src/server/routes.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/nkanaev/yarr/src/assets"
+	"github.com/nkanaev/yarr/src/content/htmlutil"
 	"github.com/nkanaev/yarr/src/content/readability"
 	"github.com/nkanaev/yarr/src/content/sanitizer"
 	"github.com/nkanaev/yarr/src/content/silo"
@@ -33,7 +34,7 @@ func (s *Server) handler() http.Handler {
 			BasePath: s.BasePath,
 			Username: s.Username,
 			Password: s.Password,
-			Public:   "/static",
+			Public:   []string{"/static", "/fever"},
 		}
 		r.Use(a.Handler)
 	}
@@ -56,6 +57,7 @@ func (s *Server) handler() http.Handler {
 	r.For("/opml/export", s.handleOPMLExport)
 	r.For("/page", s.handlePageCrawl)
 	r.For("/logout", s.handleLogout)
+	r.For("/fever/", s.handleFever)
 
 	return r
 }
@@ -312,6 +314,14 @@ func (s *Server) handleItem(c *router.Context) {
 			c.Out.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
+        // runtime fix for relative links
+        if !strings.HasPrefix(item.Link, "http") {
+            if feed := s.db.GetFeed(item.FeedId); feed != nil {
+                item.Link = htmlutil.AbsoluteUrl(item.Link, feed.Link)
+            }
+        }
+
 		item.Content = sanitizer.Sanitize(item.Link, item.Content)
 
 		c.JSON(http.StatusOK, item)
@@ -355,7 +365,7 @@ func (s *Server) handleItemList(c *router.Context) {
 		}
 		newestFirst := query.Get("oldest_first") != "true"
 
-		items := s.db.ListItems(filter, perPage+1, newestFirst)
+		items := s.db.ListItems(filter, perPage+1, newestFirst, false)
 		hasMore := false
 		if len(items) == perPage+1 {
 			hasMore = true

src/storage/item.go

@@ -62,11 +62,17 @@ type ItemFilter struct {
 	Status   *ItemStatus
 	Search   *string
 	After    *int64
+	IDs      *[]int64
+	SinceID  *int64
+	MaxID    *int64
+	Before   *time.Time
 }
 
 type MarkFilter struct {
 	FolderID *int64
 	FeedID   *int64
+
+	Before *time.Time
 }
 
 func (s *Storage) CreateItems(items []Item) bool {
@@ -140,6 +146,28 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
 		cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = ?)", compare))
 		args = append(args, *filter.After)
 	}
+	if filter.IDs != nil && len(*filter.IDs) > 0 {
+		qmarks := make([]string, len(*filter.IDs))
+		idargs := make([]interface{}, len(*filter.IDs))
+		for i, id := range *filter.IDs {
+			qmarks[i] = "?"
+			idargs[i] = id
+		}
+		cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")")
+		args = append(args, idargs...)
+	}
+	if filter.SinceID != nil {
+		cond = append(cond, "i.id > ?")
+		args = append(args, filter.SinceID)
+	}
+	if filter.MaxID != nil {
+		cond = append(cond, "i.id < ?")
+		args = append(args, filter.MaxID)
+	}
+	if filter.Before != nil {
+		cond = append(cond, "i.date < ?")
+		args = append(args, filter.Before)
+	}
 
 	predicate := "1"
 	if len(cond) > 0 {
@@ -149,7 +177,7 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
 	return predicate, args
 }
 
-func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []Item {
+func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item {
 	predicate, args := listQueryPredicate(filter, newestFirst)
 	result := make([]Item, 0, 0)
 
@@ -157,17 +185,23 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
 	if !newestFirst {
 		order = "date asc, id asc"
 	}
+	if filter.IDs != nil || filter.SinceID != nil || filter.MaxID != nil {
+		order = "i.id asc"
+	}
 
+	selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url"
+	if withContent {
+		selectCols += ", i.content"
+	} else {
+		selectCols += ", '' as content"
+	}
 	query := fmt.Sprintf(`
-		select
-			i.id, i.guid, i.feed_id,
-			i.title, i.link, i.date,
-			i.status, i.image, i.podcast_url
+		select %s
 		from items i
 		where %s
 		order by %s
 		limit %d
-		`, predicate, order, limit)
+		`, selectCols, predicate, order, limit)
 	rows, err := s.db.Query(query, args...)
 	if err != nil {
 		log.Print(err)
@@ -178,7 +212,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
 		err = rows.Scan(
 			&x.Id, &x.GUID, &x.FeedId,
 			&x.Title, &x.Link, &x.Date,
-			&x.Status, &x.ImageURL, &x.AudioURL,
+			&x.Status, &x.ImageURL, &x.AudioURL, &x.Content,
 		)
 		if err != nil {
 			log.Print(err)
@@ -214,7 +248,11 @@ func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
 }
 
 func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
-	predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID}, false)
+	predicate, args := listQueryPredicate(ItemFilter{
+		FolderID: filter.FolderID,
+		FeedID:   filter.FeedID,
+		Before:   filter.Before,
+	}, false)
 	query := fmt.Sprintf(`
 		update items as i set status = %d
 		where %s and i.status != %d

src/storage/item_test.go

@@ -104,7 +104,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by folder_id
 
-	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false))
+	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
 	want := []string{"item111", "item112", "item113", "item121", "item122"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -112,7 +112,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
 	want = []string{"item211", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -122,7 +122,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by feed_id
 
-	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
 	want = []string{"item111", "item112", "item113"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -130,7 +130,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
 	want = []string{"item011", "item012", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -141,7 +141,7 @@ func TestListItems(t *testing.T) {
 	// filter by status
 
 	var starred ItemStatus = STARRED
-	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
 	want = []string{"item113", "item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -150,7 +150,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	var unread ItemStatus = UNREAD
-	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
 	want = []string{"item111", "item121", "item011"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -160,7 +160,7 @@ func TestListItems(t *testing.T) {
 
 	// limit
 
-	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
 	want = []string{"item111", "item112"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -171,7 +171,7 @@ func TestListItems(t *testing.T) {
 	// filter by search
 	db.SyncSearch()
 	search1 := "title111"
-	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
 	want = []string{"item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -180,7 +180,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	// sort by date
-	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
 	want = []string{"item013", "item012", "item011", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -197,7 +197,7 @@ func TestListItemsPaginated(t *testing.T) {
 	item121 := getItem(db, "item121")
 
 	// all, newest first
-	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true))
+	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
 	want := []string{"item011", "item212", "item211"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -207,7 +207,7 @@ func TestListItemsPaginated(t *testing.T) {
 
 	// unread, newest first
 	unread := UNREAD
-	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true))
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false))
 	want = []string{"item011", "item121", "item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -217,7 +217,7 @@ func TestListItemsPaginated(t *testing.T) {
 
 	// starred, oldest first
 	starred := STARRED
-	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false))
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false))
 	want = []string{"item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -233,7 +233,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db1 := testDB()
 	testItemsSetup(db1)
 	db1.MarkItemsRead(MarkFilter{})
-	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false))
+	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want := []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item011", "item012",
@@ -247,7 +247,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db2 := testDB()
 	scope2 := testItemsSetup(db2)
 	db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
-	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false))
+	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item012",
@@ -261,7 +261,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db3 := testDB()
 	scope3 := testItemsSetup(db3)
 	db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
-	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false))
+	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item122",
 		"item211", "item012",
@@ -319,7 +319,7 @@ func TestDeleteOldItems(t *testing.T) {
 	}
 
 	db.DeleteOldItems()
-	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false)
+	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false, false)
 	if len(feedItems) != len(items)-3 {
 		t.Fatalf(
 			"invalid number of old items kept\nwant: %d\nhave: %d",