update changelog

doc/changelog.txt

@@ -1,4 +1,10 @@
-# upcoming
+# upcoming 
+
+- (fix) duplicate articles caused by the same feed addition (thanks to @adaszko)
+- (fix) relative article links (thanks to @adazsko for the report)
+- (fix) atom article links stored in id element (thanks to @adazsko for the report)
+
+# v2.4 (2023-08-15)
 
 - (new) ARM build support (thanks to @tillcash & @fenuks)
 - (new) auth configuration via param or env variable (thanks to @pierreprinetti)

readme.md

@@ -3,13 +3,13 @@
 **yarr** (yet another rss reader) is a web-based feed aggregator which can be used both
 as a desktop application and a personal self-hosted server.
 
-It is written in Go with the frontend in Vue.js. The storage is backed by SQLite.
+The app is a single binary with an embedded database (SQLite).
 
 ![screenshot](etc/promo.png)
 
 ## usage
 
-The latest prebuilt binaries for Linux/MacOS/Windows are available
+The latest prebuilt binaries for Linux/MacOS/Windows AMD64 are available
 [here](https://github.com/nkanaev/yarr/releases/latest).
 
 ### macos

src/content/htmlutil/urlutils.go

@@ -2,6 +2,7 @@ package htmlutil
 
 import (
 	"net/url"
+	"strings"
 )
 
 func Any(els []string, el string, match func(string, string) bool) bool {
@@ -31,3 +32,7 @@ func URLDomain(val string) string {
 	}
 	return val
 }
+
+func IsAPossibleLink(val string) bool {
+	return strings.HasPrefix(val, "http://") || strings.HasPrefix(val, "https://")
+}

src/parser/atom.go

@@ -81,9 +81,16 @@ func ParseAtom(r io.Reader) (*Feed, error) {
 		SiteURL: firstNonEmpty(srcfeed.Links.First("alternate"), srcfeed.Links.First("")),
 	}
 	for _, srcitem := range srcfeed.Entries {
-		link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""))
+		linkFromID := ""
+		guidFromID := ""
+		if htmlutil.IsAPossibleLink(srcitem.ID) {
+			linkFromID = srcitem.ID
+			guidFromID = srcitem.ID + "::" + srcitem.Updated
+		}
+
+		link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""), linkFromID)
 		dstfeed.Items = append(dstfeed.Items, Item{
-			GUID:     firstNonEmpty(srcitem.ID, link),
+			GUID:     firstNonEmpty(guidFromID, srcitem.ID, link),
 			Date:     dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)),
 			URL:      link,
 			Title:    srcitem.Title.Text(),

src/parser/atom_test.go

@@ -131,3 +131,48 @@ func TestAtomImageLinkDuplicated(t *testing.T) {
 		t.Fatal("item.image_url must be unset if present in the content")
 	}
 }
+
+func TestAtomLinkInID(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
+			<entry>
+                <title>one updated</title>
+                <id>https://example.com/posts/1</id>
+                <updated>2003-12-13T09:17:51</updated>
+			</entry>
+			<entry>
+                <title>two</title>
+                <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
+			</entry>
+			<entry>
+                <title>one</title>
+                <id>https://example.com/posts/1</id>
+			</entry>
+		</feed>
+	`))
+	have := feed.Items
+	want := []Item{
+		Item{
+			GUID:     "https://example.com/posts/1::2003-12-13T09:17:51",
+			Date:     time.Date(2003, time.December, 13, 9, 17, 51, 0, time.UTC),
+			URL:      "https://example.com/posts/1",
+			Title:    "one updated",
+        },
+		Item{
+			GUID: "urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6",
+            Date: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), URL: "",
+            Title: "two",
+        },
+        Item{
+            GUID: "https://example.com/posts/1::",
+            Date: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
+            URL: "https://example.com/posts/1",
+            Title: "one",
+            Content: "",
+        },
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Fatalf("\nwant: %#v\nhave: %#v\n", want, have)
+	}
+}

src/parser/rss.go

@@ -20,7 +20,7 @@ type rssFeed struct {
 }
 
 type rssItem struct {
-	GUID        rssGuid         `xml:"guid"`
+	GUID        rssGuid        `xml:"guid"`
 	Title       string         `xml:"title"`
 	Link        string         `xml:"rss link"`
 	Description string         `xml:"rss description"`
@@ -86,10 +86,10 @@ func ParseRSS(r io.Reader) (*Feed, error) {
 			}
 		}
 
-        permalink := ""
-        if srcitem.GUID.IsPermaLink == "true" {
-            permalink = srcitem.GUID.GUID
-        }
+		permalink := ""
+		if srcitem.GUID.IsPermaLink == "true" {
+			permalink = srcitem.GUID.GUID
+		}
 
 		dstfeed.Items = append(dstfeed.Items, Item{
 			GUID:     firstNonEmpty(srcitem.GUID.GUID, srcitem.Link),

src/parser/rss_test.go

@@ -217,11 +217,11 @@ func TestRSSIsPermalink(t *testing.T) {
 	`))
 	have := feed.Items
 	want := []Item{
-        {
-            GUID:    "http://example.com/posts/1",
-            URL:     "http://example.com/posts/1",
-        },
-    }
+		{
+			GUID: "http://example.com/posts/1",
+			URL:  "http://example.com/posts/1",
+		},
+	}
 	for i := 0; i < len(want); i++ {
 		if want[i] != have[i] {
 			t.Errorf("Failed to handle isPermalink\nwant: %#v\nhave: %#v\n", want[i], have[i])

src/server/routes.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	"github.com/nkanaev/yarr/src/assets"
+	"github.com/nkanaev/yarr/src/content/htmlutil"
 	"github.com/nkanaev/yarr/src/content/readability"
 	"github.com/nkanaev/yarr/src/content/sanitizer"
 	"github.com/nkanaev/yarr/src/content/silo"
@@ -312,6 +313,14 @@ func (s *Server) handleItem(c *router.Context) {
 			c.Out.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
+		// runtime fix for relative links
+		if !htmlutil.IsAPossibleLink(item.Link) {
+			if feed := s.db.GetFeed(item.FeedId); feed != nil {
+				item.Link = htmlutil.AbsoluteUrl(item.Link, feed.Link)
+			}
+		}
+
 		item.Content = sanitizer.Sanitize(item.Link, item.Content)
 
 		c.JSON(http.StatusOK, item)

src/storage/feed.go

@@ -20,18 +20,19 @@ func (s *Storage) CreateFeed(title, description, link, feedLink string, folderId
 	if title == "" {
 		title = feedLink
 	}
-	result, err := s.db.Exec(`
+	row := s.db.QueryRow(`
 		insert into feeds (title, description, link, feed_link, folder_id) 
 		values (?, ?, ?, ?, ?)
-		on conflict (feed_link) do update set folder_id=?`,
+		on conflict (feed_link) do update set folder_id = ?
+        returning id`,
 		title, description, link, feedLink, folderId,
 		folderId,
 	)
+
+	var id int64
+	err := row.Scan(&id)
 	if err != nil {
-		return nil
-	}
-	id, idErr := result.LastInsertId()
-	if idErr != nil {
+		log.Print(err)
 		return nil
 	}
 	return &Feed{

src/storage/feed_test.go

@@ -17,6 +17,23 @@ func TestCreateFeed(t *testing.T) {
 	}
 }
 
+func TestCreateFeedSameLink(t *testing.T) {
+	db := testDB()
+	feed1 := db.CreateFeed("title", "", "", "http://example1.com/feed.xml", nil)
+	if feed1 == nil || feed1.Id == 0 {
+		t.Fatal("expected feed")
+	}
+
+	for i := 0; i < 10; i++ {
+		db.CreateFeed("title", "", "", "http://example2.com/feed.xml", nil)
+	}
+
+	feed2 := db.CreateFeed("title", "", "http://example.com", "http://example1.com/feed.xml", nil)
+	if feed1.Id != feed2.Id {
+		t.Fatalf("expected the same feed.\nwant: %#v\nhave: %#v", feed1, feed2)
+	}
+}
+
 func TestReadFeed(t *testing.T) {
 	db := testDB()
 	if db.GetFeed(100500) != nil {

src/storage/folder.go

@@ -1,7 +1,6 @@
 package storage
 
 import (
-	"fmt"
 	"log"
 )
 
@@ -13,35 +12,21 @@ type Folder struct {
 
 func (s *Storage) CreateFolder(title string) *Folder {
 	expanded := true
-	result, err := s.db.Exec(`
+	row := s.db.QueryRow(`
 		insert into folders (title, is_expanded) values (?, ?)
-		on conflict (title) do nothing`,
+		on conflict (title) do update set title = ?
+        returning id`,
 		title, expanded,
+		// provide title again so that we can extract row id
+		title,
 	)
-	if err != nil {
-		fmt.Println(err)
-		return nil
-	}
-
 	var id int64
-	numrows, err := result.RowsAffected()
+	err := row.Scan(&id)
+
 	if err != nil {
 		log.Print(err)
 		return nil
 	}
-	if numrows == 1 {
-		id, err = result.LastInsertId()
-		if err != nil {
-			log.Print(err)
-			return nil
-		}
-	} else {
-		err = s.db.QueryRow(`select id, is_expanded from folders where title=?`, title).Scan(&id, &expanded)
-		if err != nil {
-			log.Print(err)
-			return nil
-		}
-	}
 	return &Folder{Id: id, Title: title, IsExpanded: expanded}
 }