Update readme.md

Fever API spec: https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md

doc/changelog.txt

@@ -1,5 +1,12 @@
 # upcoming 
 
+- (new) Fever API support (thanks to @icefed)
+- (fix) duplicate articles caused by the same feed addition (thanks to @adaszko)
+- (fix) relative article links (thanks to @adazsko for the report)
+- (fix) atom article links stored in id element (thanks to @adazsko for the report)
+
+# v2.4 (2023-08-15)
+
 - (new) ARM build support (thanks to @tillcash & @fenuks)
 - (new) auth configuration via param or env variable (thanks to @pierreprinetti)
 - (new) web app manifest for an app-like experience on mobile (thanks to @qbit)

fever.md

@@ -0,0 +1,19 @@
+# Fever API support
+
+Fever API is a kind of RSS HTTP API interface, because the Fever API definition is not very clear, so the implementation of Fever server and Client may have some compatibility problems.
+
+The Fever API implemented by Yarr is based on the Fever API spec: https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md.
+
+Here are some Apps that have been tested to work with yarr.  Feel free to test other Clients/Apps and update the list here.
+
+>  Different apps support different URL/Address formats.  Please note whether the URL entered has `http://` scheme and `/` suffix.
+
+| App                                                                       | Platforms        | Config Server URL                                   |
+|:------------------------------------------------------------------------- | ---------------- |:--------------------------------------------------- |
+| [Reeder](https://reederapp.com/)                                          | MacOS<br>iOS     | 127.0.0.1:7070/fever<br>http://127.0.0.1:7070/fever |
+| [ReadKit](https://readkit.app/)                                           | MacOS<br>iOS     | http://127.0.0.1:7070/fever                         |
+| [Fluent Reader](https://github.com/yang991178/fluent-reader)              | MacOS<br>Windows | http://127.0.0.1:7070/fever/                        |
+| [Unread](https://apps.apple.com/us/app/unread-an-rss-reader/id1363637349) | iOS              | http://127.0.0.1:7070/fever                         |
+| [Fiery Feeds](https://voidstern.net/fiery-feeds)                          | MacOS<br>iOS     | http://127.0.0.1:7070/fever                         |
+
+If you are having trouble using Fever, please open an issue and @icefed, thanks.

readme.md

@@ -3,13 +3,13 @@
 **yarr** (yet another rss reader) is a web-based feed aggregator which can be used both
 as a desktop application and a personal self-hosted server.
 
-It is written in Go with the frontend in Vue.js. The storage is backed by SQLite.
+The app is a single binary with an embedded database (SQLite).
 
 ![screenshot](etc/promo.png)
 
 ## usage
 
-The latest prebuilt binaries for Linux/MacOS/Windows are available
+The latest prebuilt binaries for Linux/MacOS/Windows AMD64 are available
 [here](https://github.com/nkanaev/yarr/releases/latest).
 
 ### macos
@@ -30,6 +30,8 @@ and run [the script](etc/install-linux.sh).
 For self-hosting, see `yarr -h` for auth, tls & server configuration flags.
 For building from source code, see [build.md](build.md)
 
+For Fever API support, see [fever.md](fever.md).
+
 ## credits
 
 [Feather](http://feathericons.com/) for icons.

src/content/htmlutil/urlutils.go

@@ -2,6 +2,7 @@ package htmlutil
 
 import (
 	"net/url"
+	"strings"
 )
 
 func Any(els []string, el string, match func(string, string) bool) bool {
@@ -31,3 +32,7 @@ func URLDomain(val string) string {
 	}
 	return val
 }
+
+func IsAPossibleLink(val string) bool {
+	return strings.HasPrefix(val, "http://") || strings.HasPrefix(val, "https://")
+}

src/parser/atom.go

@@ -81,9 +81,16 @@ func ParseAtom(r io.Reader) (*Feed, error) {
 		SiteURL: firstNonEmpty(srcfeed.Links.First("alternate"), srcfeed.Links.First("")),
 	}
 	for _, srcitem := range srcfeed.Entries {
-		link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""))
+		linkFromID := ""
+		guidFromID := ""
+		if htmlutil.IsAPossibleLink(srcitem.ID) {
+			linkFromID = srcitem.ID
+			guidFromID = srcitem.ID + "::" + srcitem.Updated
+		}
+
+		link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""), linkFromID)
 		dstfeed.Items = append(dstfeed.Items, Item{
-			GUID:     firstNonEmpty(srcitem.ID, link),
+			GUID:     firstNonEmpty(guidFromID, srcitem.ID, link),
 			Date:     dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)),
 			URL:      link,
 			Title:    srcitem.Title.Text(),

src/parser/atom_test.go

@@ -131,3 +131,48 @@ func TestAtomImageLinkDuplicated(t *testing.T) {
 		t.Fatal("item.image_url must be unset if present in the content")
 	}
 }
+
+func TestAtomLinkInID(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
+			<entry>
+                <title>one updated</title>
+                <id>https://example.com/posts/1</id>
+                <updated>2003-12-13T09:17:51</updated>
+			</entry>
+			<entry>
+                <title>two</title>
+                <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
+			</entry>
+			<entry>
+                <title>one</title>
+                <id>https://example.com/posts/1</id>
+			</entry>
+		</feed>
+	`))
+	have := feed.Items
+	want := []Item{
+		Item{
+			GUID:     "https://example.com/posts/1::2003-12-13T09:17:51",
+			Date:     time.Date(2003, time.December, 13, 9, 17, 51, 0, time.UTC),
+			URL:      "https://example.com/posts/1",
+			Title:    "one updated",
+        },
+		Item{
+			GUID: "urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6",
+            Date: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), URL: "",
+            Title: "two",
+        },
+        Item{
+            GUID: "https://example.com/posts/1::",
+            Date: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
+            URL: "https://example.com/posts/1",
+            Title: "one",
+            Content: "",
+        },
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Fatalf("\nwant: %#v\nhave: %#v\n", want, have)
+	}
+}

src/parser/rss.go

@@ -20,7 +20,7 @@ type rssFeed struct {
 }
 
 type rssItem struct {
-	GUID        rssGuid         `xml:"guid"`
+	GUID        rssGuid        `xml:"guid"`
 	Title       string         `xml:"title"`
 	Link        string         `xml:"rss link"`
 	Description string         `xml:"rss description"`
@@ -86,10 +86,10 @@ func ParseRSS(r io.Reader) (*Feed, error) {
 			}
 		}
 
-        permalink := ""
-        if srcitem.GUID.IsPermaLink == "true" {
-            permalink = srcitem.GUID.GUID
-        }
+		permalink := ""
+		if srcitem.GUID.IsPermaLink == "true" {
+			permalink = srcitem.GUID.GUID
+		}
 
 		dstfeed.Items = append(dstfeed.Items, Item{
 			GUID:     firstNonEmpty(srcitem.GUID.GUID, srcitem.Link),

src/parser/rss_test.go

@@ -217,11 +217,11 @@ func TestRSSIsPermalink(t *testing.T) {
 	`))
 	have := feed.Items
 	want := []Item{
-        {
-            GUID:    "http://example.com/posts/1",
-            URL:     "http://example.com/posts/1",
-        },
-    }
+		{
+			GUID: "http://example.com/posts/1",
+			URL:  "http://example.com/posts/1",
+		},
+	}
 	for i := 0; i < len(want); i++ {
 		if want[i] != have[i] {
 			t.Errorf("Failed to handle isPermalink\nwant: %#v\nhave: %#v\n", want[i], have[i])

src/server/auth/middleware.go

@@ -12,7 +12,7 @@ type Middleware struct {
 	Username string
 	Password string
 	BasePath string
-	Public   string
+	Public   []string
 }
 
 func unsafeMethod(method string) bool {
@@ -20,9 +20,11 @@ func unsafeMethod(method string) bool {
 }
 
 func (m *Middleware) Handler(c *router.Context) {
-	if strings.HasPrefix(c.Req.URL.Path, m.BasePath+m.Public) {
-		c.Next()
-		return
+	for _, path := range m.Public {
+		if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) {
+			c.Next()
+			return
+		}
 	}
 	if IsAuthenticated(c.Req, m.Username, m.Password) {
 		c.Next()

src/server/fever.go

@@ -0,0 +1,393 @@
+package server
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/nkanaev/yarr/src/server/auth"
+	"github.com/nkanaev/yarr/src/server/router"
+	"github.com/nkanaev/yarr/src/storage"
+)
+
+type FeverGroup struct {
+	ID    int64  `json:"id"`
+	Title string `json:"title"`
+}
+
+type FeverFeedsGroup struct {
+	GroupID int64  `json:"group_id"`
+	FeedIDs string `json:"feed_ids"`
+}
+
+type FeverFeed struct {
+	ID          int64  `json:"id"`
+	FaviconID   int64  `json:"favicon_id"`
+	Title       string `json:"title"`
+	Url         string `json:"url"`
+	SiteUrl     string `json:"site_url"`
+	IsSpark     int    `json:"is_spark"`
+	LastUpdated int64  `json:"last_updated_on_time"`
+}
+
+type FeverItem struct {
+	ID        int64  `json:"id"`
+	FeedID    int64  `json:"feed_id"`
+	Title     string `json:"title"`
+	Author    string `json:"author"`
+	HTML      string `json:"html"`
+	Url       string `json:"url"`
+	IsSaved   int    `json:"is_saved"`
+	IsRead    int    `json:"is_read"`
+	CreatedAt int64  `json:"created_on_time"`
+}
+
+type FeverFavicon struct {
+	ID   int64  `json:"id"`
+	Data string `json:"data"`
+}
+
+func writeFeverJSON(c *router.Context, data map[string]interface{}, lastRefreshed int64) {
+	data["api_version"] = 3
+	data["auth"] = 1
+	data["last_refreshed_on_time"] = lastRefreshed
+	c.JSON(http.StatusOK, data)
+}
+
+func getLastRefreshedOnTime(httpStates map[int64]storage.HTTPState) int64 {
+	if len(httpStates) == 0 {
+		return 0
+	}
+
+	var lastRefreshed int64
+	for _, state := range httpStates {
+		if state.LastRefreshed.Unix() > lastRefreshed {
+			lastRefreshed = state.LastRefreshed.Unix()
+		}
+	}
+	return lastRefreshed
+}
+
+func (s *Server) feverAuth(c *router.Context) bool {
+	if s.Username != "" && s.Password != "" {
+		apiKey := c.Req.FormValue("api_key")
+		apiKey = strings.ToLower(apiKey)
+		md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password)))
+		hexMD5HashValue := fmt.Sprintf("%x", md5HashValue[:])
+		if !auth.StringsEqual(apiKey, hexMD5HashValue) {
+			return false
+		}
+	}
+	return true
+}
+
+func formHasValue(values url.Values, value string) bool {
+	if _, ok := values[value]; ok {
+		return true
+	}
+	return false
+}
+
+func (s *Server) handleFever(c *router.Context) {
+	c.Req.ParseForm()
+	if !s.feverAuth(c) {
+		c.JSON(http.StatusOK, map[string]interface{}{
+			"api_version":            3,
+			"auth":                   0,
+			"last_refreshed_on_time": 0,
+		})
+		return
+	}
+
+	switch {
+	case formHasValue(c.Req.Form, "groups"):
+		s.feverGroupsHandler(c)
+	case formHasValue(c.Req.Form, "feeds"):
+		s.feverFeedsHandler(c)
+	case formHasValue(c.Req.Form, "unread_item_ids"):
+		s.feverUnreadItemIDsHandler(c)
+	case formHasValue(c.Req.Form, "saved_item_ids"):
+		s.feverSavedItemIDsHandler(c)
+	case formHasValue(c.Req.Form, "favicons"):
+		s.feverFaviconsHandler(c)
+	case formHasValue(c.Req.Form, "items"):
+		s.feverItemsHandler(c)
+	case formHasValue(c.Req.Form, "links"):
+		s.feverLinksHandler(c)
+	case formHasValue(c.Req.Form, "mark"):
+		s.feverMarkHandler(c)
+	default:
+		c.JSON(http.StatusOK, map[string]interface{}{
+			"api_version":            3,
+			"auth":                   1,
+			"last_refreshed_on_time": getLastRefreshedOnTime(s.db.ListHTTPStates()),
+		})
+	}
+}
+
+func joinInts(values []int64) string {
+	var result strings.Builder
+	for i, val := range values {
+		fmt.Fprintf(&result, "%d", val)
+		if i != len(values)-1 {
+			result.WriteString(",")
+		}
+	}
+	return result.String()
+}
+
+func feedGroups(db *storage.Storage) []*FeverFeedsGroup {
+	feeds := db.ListFeeds()
+
+	groupFeeds := make(map[int64][]int64)
+	for _, feed := range feeds {
+		if feed.FolderId == nil {
+			continue
+		}
+		groupFeeds[*feed.FolderId] = append(groupFeeds[*feed.FolderId], feed.Id)
+	}
+	result := make([]*FeverFeedsGroup, 0)
+	for groupId, feedIds := range groupFeeds {
+		result = append(result, &FeverFeedsGroup{
+			GroupID: groupId,
+			FeedIDs: joinInts(feedIds),
+		})
+	}
+	return result
+}
+
+func (s *Server) feverGroupsHandler(c *router.Context) {
+	folders := s.db.ListFolders()
+	groups := make([]*FeverGroup, len(folders))
+	for i, folder := range folders {
+		groups[i] = &FeverGroup{ID: folder.Id, Title: folder.Title}
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"groups":       groups,
+		"feeds_groups": feedGroups(s.db),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverFeedsHandler(c *router.Context) {
+	feeds := s.db.ListFeeds()
+	httpStates := s.db.ListHTTPStates()
+
+	feverFeeds := make([]*FeverFeed, len(feeds))
+	for i, feed := range feeds {
+		var lastUpdated int64
+		if state, ok := httpStates[feed.Id]; ok {
+			lastUpdated = state.LastRefreshed.Unix()
+		}
+		feverFeeds[i] = &FeverFeed{
+			ID:          feed.Id,
+			FaviconID:   feed.Id,
+			Title:       feed.Title,
+			Url:         feed.FeedLink,
+			SiteUrl:     feed.Link,
+			IsSpark:     0,
+			LastUpdated: lastUpdated,
+		}
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"feeds":        feverFeeds,
+		"feeds_groups": feedGroups(s.db),
+	}, getLastRefreshedOnTime(httpStates))
+}
+
+func (s *Server) feverFaviconsHandler(c *router.Context) {
+	feeds := s.db.ListFeeds()
+	favicons := make([]*FeverFavicon, len(feeds))
+	for i, feed := range feeds {
+		data := "data:image/gif;base64,R0lGODlhAQABAAAAACw="
+		if feed.HasIcon {
+			icon := s.db.GetFeed(feed.Id).Icon
+			data = fmt.Sprintf(
+				"data:%s;base64,%s",
+				http.DetectContentType(*icon),
+				base64.StdEncoding.EncodeToString(*icon),
+			)
+		}
+		favicons[i] = &FeverFavicon{ID: feed.Id, Data: data}
+	}
+
+	writeFeverJSON(c, map[string]interface{}{
+		"favicons": favicons,
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+// for memory pressure reasons, we only return a limited number of items
+// documented at https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md#items
+const listLimit = 50
+
+func (s *Server) feverItemsHandler(c *router.Context) {
+	filter := storage.ItemFilter{}
+	query := c.Req.URL.Query()
+
+	switch {
+	case query.Get("with_ids") != "":
+		ids := make([]int64, 0)
+		for _, idstr := range strings.Split(query.Get("with_ids"), ",") {
+			if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+				ids = append(ids, idnum)
+			}
+		}
+		filter.IDs = &ids
+	case query.Get("since_id") != "":
+		idstr := query.Get("since_id")
+		if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+			filter.SinceID = &idnum
+		}
+	case query.Get("max_id") != "":
+		idstr := query.Get("max_id")
+		if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+			filter.MaxID = &idnum
+		}
+	}
+
+	items := s.db.ListItems(filter, listLimit, true, true)
+
+	feverItems := make([]FeverItem, len(items))
+	for i, item := range items {
+		date := item.Date
+		time := date.Unix()
+
+		isSaved := 0
+		if item.Status == storage.STARRED {
+			isSaved = 1
+		}
+		isRead := 0
+		if item.Status == storage.READ {
+			isRead = 1
+		}
+		feverItems[i] = FeverItem{
+			ID:        item.Id,
+			FeedID:    item.FeedId,
+			Title:     item.Title,
+			Author:    "",
+			HTML:      item.Content,
+			Url:       item.Link,
+			IsSaved:   isSaved,
+			IsRead:    isRead,
+			CreatedAt: time,
+		}
+	}
+
+	totalItems := s.db.CountItems(storage.ItemFilter{})
+
+	writeFeverJSON(c, map[string]interface{}{
+		"items":       feverItems,
+		"total_items": totalItems,
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverLinksHandler(c *router.Context) {
+	writeFeverJSON(c, map[string]interface{}{
+		"links": make([]interface{}, 0),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
+	status := storage.UNREAD
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"unread_item_ids": joinInts(itemIds),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverSavedItemIDsHandler(c *router.Context) {
+	status := storage.STARRED
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"saved_item_ids": joinInts(itemIds),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverMarkHandler(c *router.Context) {
+	id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64)
+	if err != nil {
+		log.Print("invalid id:", err)
+		return
+	}
+
+	switch c.Req.Form.Get("mark") {
+	case "item":
+		var status storage.ItemStatus
+		switch c.Req.Form.Get("as") {
+		case "read":
+			status = storage.READ
+		case "unread":
+			status = storage.UNREAD
+		case "saved":
+			status = storage.STARRED
+		case "unsaved":
+			status = storage.READ
+		default:
+			c.Out.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		s.db.UpdateItemStatus(id, status)
+	case "feed":
+		if c.Req.Form.Get("as") != "read" {
+			c.Out.WriteHeader(http.StatusBadRequest)
+		}
+		markFilter := storage.MarkFilter{FeedID: &id}
+		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
+			before := time.Unix(x, 0)
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
+	case "group":
+		if c.Req.Form.Get("as") != "read" {
+			c.Out.WriteHeader(http.StatusBadRequest)
+		}
+		markFilter := storage.MarkFilter{FolderID: &id}
+		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
+			before := time.Unix(x, 0)
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
+	default:
+		c.Out.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	c.JSON(http.StatusOK, map[string]interface{}{
+		"api_version": 3,
+		"auth":        1,
+	})
+}

src/server/routes.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/nkanaev/yarr/src/assets"
+	"github.com/nkanaev/yarr/src/content/htmlutil"
 	"github.com/nkanaev/yarr/src/content/readability"
 	"github.com/nkanaev/yarr/src/content/sanitizer"
 	"github.com/nkanaev/yarr/src/content/silo"
@@ -33,7 +34,7 @@ func (s *Server) handler() http.Handler {
 			BasePath: s.BasePath,
 			Username: s.Username,
 			Password: s.Password,
-			Public:   "/static",
+			Public:   []string{"/static", "/fever"},
 		}
 		r.Use(a.Handler)
 	}
@@ -56,6 +57,7 @@ func (s *Server) handler() http.Handler {
 	r.For("/opml/export", s.handleOPMLExport)
 	r.For("/page", s.handlePageCrawl)
 	r.For("/logout", s.handleLogout)
+	r.For("/fever/", s.handleFever)
 
 	return r
 }
@@ -312,6 +314,14 @@ func (s *Server) handleItem(c *router.Context) {
 			c.Out.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
+		// runtime fix for relative links
+		if !htmlutil.IsAPossibleLink(item.Link) {
+			if feed := s.db.GetFeed(item.FeedId); feed != nil {
+				item.Link = htmlutil.AbsoluteUrl(item.Link, feed.Link)
+			}
+		}
+
 		item.Content = sanitizer.Sanitize(item.Link, item.Content)
 
 		c.JSON(http.StatusOK, item)
@@ -355,7 +365,7 @@ func (s *Server) handleItemList(c *router.Context) {
 		}
 		newestFirst := query.Get("oldest_first") != "true"
 
-		items := s.db.ListItems(filter, perPage+1, newestFirst)
+		items := s.db.ListItems(filter, perPage+1, newestFirst, false)
 		hasMore := false
 		if len(items) == perPage+1 {
 			hasMore = true

src/storage/feed.go

@@ -20,18 +20,19 @@ func (s *Storage) CreateFeed(title, description, link, feedLink string, folderId
 	if title == "" {
 		title = feedLink
 	}
-	result, err := s.db.Exec(`
+	row := s.db.QueryRow(`
 		insert into feeds (title, description, link, feed_link, folder_id) 
 		values (?, ?, ?, ?, ?)
-		on conflict (feed_link) do update set folder_id=?`,
+		on conflict (feed_link) do update set folder_id = ?
+        returning id`,
 		title, description, link, feedLink, folderId,
 		folderId,
 	)
+
+	var id int64
+	err := row.Scan(&id)
 	if err != nil {
-		return nil
-	}
-	id, idErr := result.LastInsertId()
-	if idErr != nil {
+		log.Print(err)
 		return nil
 	}
 	return &Feed{

src/storage/feed_test.go

@@ -17,6 +17,23 @@ func TestCreateFeed(t *testing.T) {
 	}
 }
 
+func TestCreateFeedSameLink(t *testing.T) {
+	db := testDB()
+	feed1 := db.CreateFeed("title", "", "", "http://example1.com/feed.xml", nil)
+	if feed1 == nil || feed1.Id == 0 {
+		t.Fatal("expected feed")
+	}
+
+	for i := 0; i < 10; i++ {
+		db.CreateFeed("title", "", "", "http://example2.com/feed.xml", nil)
+	}
+
+	feed2 := db.CreateFeed("title", "", "http://example.com", "http://example1.com/feed.xml", nil)
+	if feed1.Id != feed2.Id {
+		t.Fatalf("expected the same feed.\nwant: %#v\nhave: %#v", feed1, feed2)
+	}
+}
+
 func TestReadFeed(t *testing.T) {
 	db := testDB()
 	if db.GetFeed(100500) != nil {

src/storage/folder.go

@@ -1,7 +1,6 @@
 package storage
 
 import (
-	"fmt"
 	"log"
 )
 
@@ -13,35 +12,21 @@ type Folder struct {
 
 func (s *Storage) CreateFolder(title string) *Folder {
 	expanded := true
-	result, err := s.db.Exec(`
+	row := s.db.QueryRow(`
 		insert into folders (title, is_expanded) values (?, ?)
-		on conflict (title) do nothing`,
+		on conflict (title) do update set title = ?
+        returning id`,
 		title, expanded,
+		// provide title again so that we can extract row id
+		title,
 	)
-	if err != nil {
-		fmt.Println(err)
-		return nil
-	}
-
 	var id int64
-	numrows, err := result.RowsAffected()
+	err := row.Scan(&id)
+
 	if err != nil {
 		log.Print(err)
 		return nil
 	}
-	if numrows == 1 {
-		id, err = result.LastInsertId()
-		if err != nil {
-			log.Print(err)
-			return nil
-		}
-	} else {
-		err = s.db.QueryRow(`select id, is_expanded from folders where title=?`, title).Scan(&id, &expanded)
-		if err != nil {
-			log.Print(err)
-			return nil
-		}
-	}
 	return &Folder{Id: id, Title: title, IsExpanded: expanded}
 }
 

src/storage/item.go

@@ -62,11 +62,17 @@ type ItemFilter struct {
 	Status   *ItemStatus
 	Search   *string
 	After    *int64
+	IDs      *[]int64
+	SinceID  *int64
+	MaxID    *int64
+	Before   *time.Time
 }
 
 type MarkFilter struct {
 	FolderID *int64
 	FeedID   *int64
+
+	Before *time.Time
 }
 
 func (s *Storage) CreateItems(items []Item) bool {
@@ -140,6 +146,28 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
 		cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = ?)", compare))
 		args = append(args, *filter.After)
 	}
+	if filter.IDs != nil && len(*filter.IDs) > 0 {
+		qmarks := make([]string, len(*filter.IDs))
+		idargs := make([]interface{}, len(*filter.IDs))
+		for i, id := range *filter.IDs {
+			qmarks[i] = "?"
+			idargs[i] = id
+		}
+		cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")")
+		args = append(args, idargs...)
+	}
+	if filter.SinceID != nil {
+		cond = append(cond, "i.id > ?")
+		args = append(args, filter.SinceID)
+	}
+	if filter.MaxID != nil {
+		cond = append(cond, "i.id < ?")
+		args = append(args, filter.MaxID)
+	}
+	if filter.Before != nil {
+		cond = append(cond, "i.date < ?")
+		args = append(args, filter.Before)
+	}
 
 	predicate := "1"
 	if len(cond) > 0 {
@@ -149,7 +177,24 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
 	return predicate, args
 }
 
-func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []Item {
+func (s *Storage) CountItems(filter ItemFilter) int {
+	predicate, args := listQueryPredicate(filter, false)
+
+	var count int
+	query := fmt.Sprintf(`
+		select count(*)
+		from items
+		where %s
+		`, predicate)
+	err := s.db.QueryRow(query, args...).Scan(&count)
+	if err != nil {
+		log.Print(err)
+		return 0
+	}
+	return count
+}
+
+func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item {
 	predicate, args := listQueryPredicate(filter, newestFirst)
 	result := make([]Item, 0, 0)
 
@@ -157,17 +202,26 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
 	if !newestFirst {
 		order = "date asc, id asc"
 	}
+	if filter.IDs != nil || filter.SinceID != nil {
+		order = "i.id asc"
+	}
+	if filter.MaxID != nil {
+		order = "i.id desc"
+	}
 
+	selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url"
+	if withContent {
+		selectCols += ", i.content"
+	} else {
+		selectCols += ", '' as content"
+	}
 	query := fmt.Sprintf(`
-		select
-			i.id, i.guid, i.feed_id,
-			i.title, i.link, i.date,
-			i.status, i.image, i.podcast_url
+		select %s
 		from items i
 		where %s
 		order by %s
 		limit %d
-		`, predicate, order, limit)
+		`, selectCols, predicate, order, limit)
 	rows, err := s.db.Query(query, args...)
 	if err != nil {
 		log.Print(err)
@@ -178,7 +232,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
 		err = rows.Scan(
 			&x.Id, &x.GUID, &x.FeedId,
 			&x.Title, &x.Link, &x.Date,
-			&x.Status, &x.ImageURL, &x.AudioURL,
+			&x.Status, &x.ImageURL, &x.AudioURL, &x.Content,
 		)
 		if err != nil {
 			log.Print(err)
@@ -214,7 +268,11 @@ func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
 }
 
 func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
-	predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID}, false)
+	predicate, args := listQueryPredicate(ItemFilter{
+		FolderID: filter.FolderID,
+		FeedID:   filter.FeedID,
+		Before:   filter.Before,
+	}, false)
 	query := fmt.Sprintf(`
 		update items as i set status = %d
 		where %s and i.status != %d

src/storage/item_test.go

@@ -104,7 +104,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by folder_id
 
-	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false))
+	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
 	want := []string{"item111", "item112", "item113", "item121", "item122"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -112,7 +112,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
 	want = []string{"item211", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -122,7 +122,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by feed_id
 
-	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
 	want = []string{"item111", "item112", "item113"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -130,7 +130,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
 	want = []string{"item011", "item012", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -141,7 +141,7 @@ func TestListItems(t *testing.T) {
 	// filter by status
 
 	var starred ItemStatus = STARRED
-	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
 	want = []string{"item113", "item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -150,7 +150,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	var unread ItemStatus = UNREAD
-	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
 	want = []string{"item111", "item121", "item011"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -160,7 +160,7 @@ func TestListItems(t *testing.T) {
 
 	// limit
 
-	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
 	want = []string{"item111", "item112"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -171,7 +171,7 @@ func TestListItems(t *testing.T) {
 	// filter by search
 	db.SyncSearch()
 	search1 := "title111"
-	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
 	want = []string{"item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -180,7 +180,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	// sort by date
-	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
 	want = []string{"item013", "item012", "item011", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -197,7 +197,7 @@ func TestListItemsPaginated(t *testing.T) {
 	item121 := getItem(db, "item121")
 
 	// all, newest first
-	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true))
+	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
 	want := []string{"item011", "item212", "item211"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -207,7 +207,7 @@ func TestListItemsPaginated(t *testing.T) {
 
 	// unread, newest first
 	unread := UNREAD
-	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true))
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false))
 	want = []string{"item011", "item121", "item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -217,7 +217,7 @@ func TestListItemsPaginated(t *testing.T) {
 
 	// starred, oldest first
 	starred := STARRED
-	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false))
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false))
 	want = []string{"item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -233,7 +233,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db1 := testDB()
 	testItemsSetup(db1)
 	db1.MarkItemsRead(MarkFilter{})
-	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false))
+	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want := []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item011", "item012",
@@ -247,7 +247,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db2 := testDB()
 	scope2 := testItemsSetup(db2)
 	db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
-	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false))
+	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item012",
@@ -261,7 +261,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db3 := testDB()
 	scope3 := testItemsSetup(db3)
 	db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
-	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false))
+	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item122",
 		"item211", "item012",
@@ -319,7 +319,7 @@ func TestDeleteOldItems(t *testing.T) {
 	}
 
 	db.DeleteOldItems()
-	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false)
+	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false, false)
 	if len(feedItems) != len(items)-3 {
 		t.Fatalf(
 			"invalid number of old items kept\nwant: %d\nhave: %d",