v2.3

found in: https://rss.nytimes.com/services/xml/rss/nyt/Arts.xml

when both rss and atom link elements are present, xml parser returns
empty string. provide default namespace to capture rss link properly.

doc/changelog.txt

@@ -1,5 +1,14 @@
 # upcoming
 
+- (fix) handling encodings (thanks to @f100024 & @fserb)
+- (fix) parsing xml feeds with illegal characters (thanks to @stepelu for the report)
+- (fix) old articles reappearing as unread (thanks to @adaszko for the report)
+- (fix) item list scrolling issue on large screens (thanks to @bielej for the report)
+- (fix) keyboard shortcuts color in dark mode (thanks to @John09f9 for the report)
+- (etc) autofocus when adding a new feed (thanks to @lakuapik)
+
+# v2.2 (2021-11-20)
+
 - (fix) windows console support (thanks to @dufferzafar for the report)
 - (fix) remove html tags from article titles (thanks to Alex Went for the report)
 - (etc) autoselect current folder when adding a new feed (thanks to @krkk)

doc/todo.txt

@@ -1 +0,0 @@
-- feedlist keyboard navigation is flaky in "unread" section

makefile

@@ -1,4 +1,4 @@
-VERSION=2.2
+VERSION=2.3
 GITHASH=$(shell git rev-parse --short=8 HEAD)
 
 CGO_ENABLED=1

src/assets/assets.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/fs"
 	"io/ioutil"
+	"log"
 	"os"
 )
 
@@ -29,9 +30,18 @@ func Template(path string) *template.Template {
 	if !found {
 		tmpl = template.Must(template.New(path).Delims("{%", "%}").Funcs(template.FuncMap{
 			"inline": func(svg string) template.HTML {
-				svgfile, _ := FS.Open("graphicarts/" + svg)
-				content, _ := ioutil.ReadAll(svgfile)
-				svgfile.Close()
+				svgfile, err := FS.Open("graphicarts/" + svg)
+				// should never happen
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer svgfile.Close()
+
+				content, err := ioutil.ReadAll(svgfile)
+				// should never happen
+				if err != nil {
+					log.Fatal(err)
+				}
 				return template.HTML(content)
 			},
 		}).ParseFS(FS, path))

src/assets/graphicarts/chevron-down.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-chevron-down"><polyline points="6 9 12 15 18 9"></polyline></svg>

src/assets/graphicarts/list.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-list"><line x1="8" y1="6" x2="21" y2="6"></line><line x1="8" y1="12" x2="21" y2="12"></line><line x1="8" y1="18" x2="21" y2="18"></line><line x1="3" y1="6" x2="3.01" y2="6"></line><line x1="3" y1="12" x2="3.01" y2="12"></line><line x1="3" y1="18" x2="3.01" y2="18"></line></svg>

src/assets/graphicarts/menu.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-menu"><line x1="3" y1="12" x2="21" y2="12"></line><line x1="3" y1="6" x2="21" y2="6"></line><line x1="3" y1="18" x2="21" y2="18"></line></svg>

src/assets/graphicarts/more-vertical.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-more-vertical"><circle cx="12" cy="12" r="1"></circle><circle cx="12" cy="5" r="1"></circle><circle cx="12" cy="19" r="1"></circle></svg>

src/assets/graphicarts/settings.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-settings"><circle cx="12" cy="12" r="3"></circle><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"></path></svg>

src/assets/graphicarts/trash-2.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-trash-2"><polyline points="3 6 5 6 21 6"></polyline><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"></path><line x1="10" y1="11" x2="10" y2="17"></line><line x1="14" y1="11" x2="14" y2="17"></line></svg>

src/assets/index.html

@@ -354,7 +354,7 @@
                 <p class="cursor-default"><b>New Feed</b></p>
                 <form action="" @submit.prevent="createFeed(event)" class="mt-4">
                     <label for="feed-url">URL</label>
-                    <input id="feed-url" name="url" type="url" class="form-control" required autocomplete="off" :readonly="feedNewChoice.length > 0">
+                    <input id="feed-url" name="url" type="url" class="form-control" required autocomplete="off" :readonly="feedNewChoice.length > 0" placeholder="https://example.com/feed" v-focus>
                     <label for="feed-folder" class="mt-3 d-block">
                         Folder
                         <a href="#" class="float-right text-decoration-none" @click.prevent="createNewFeedFolder()">new folder</a>

src/assets/javascripts/app.js

@@ -21,6 +21,12 @@ Vue.directive('scroll', {
   },
 })
 
+Vue.directive('focus', {
+  inserted: function(el) {
+    el.focus()
+  }
+})
+
 Vue.component('drag', {
   props: ['width'],
   template: '<div class="drag"></div>',
@@ -413,7 +419,7 @@ var vm = new Vue({
       }
 
       this.loading.items = true
-      return api.items.list(query).then(function(data) {
+      api.items.list(query).then(function(data) {
         if (loadMore) {
           vm.items = vm.items.concat(data.list)
         } else {
@@ -421,14 +427,24 @@ var vm = new Vue({
         }
         vm.itemsHasMore = data.has_more
         vm.loading.items = false
+
+        // load more if there's some space left at the bottom of the item list.
+        vm.$nextTick(function() {
+          if (vm.itemsHasMore && !vm.loading.items && vm.itemListCloseToBottom()) {
+            vm.refreshItems(true)
+          }
+        })
       })
     },
+    itemListCloseToBottom: function() {
+      var el = this.$refs.itemlist
+      var closeToBottom = (el.scrollHeight - el.scrollTop - el.offsetHeight) < 50
+      return closeToBottom
+    },
     loadMoreItems: function(event, el) {
       if (!this.itemsHasMore) return
-
       if (this.loading.items) return
-      var closeToBottom = (el.scrollHeight - el.scrollTop - el.offsetHeight) < 50
-      if (closeToBottom) this.refreshItems(true)
+      if (this.itemListCloseToBottom()) this.refreshItems(true)
     },
     markItemsRead: function() {
       var query = this.getItemsQuery()

src/assets/stylesheets/app.css

@@ -85,6 +85,10 @@ select.form-control:not([multiple]):not([size]) {
   outline: none;
 }
 
+.table-compact {
+  color: unset !important;
+}
+
 .table-compact tr td:first-child {
   padding-left: 0;
 }

src/parser/feed.go

@@ -11,18 +11,23 @@ import (
 	"time"
 
 	"github.com/nkanaev/yarr/src/content/htmlutil"
+	"golang.org/x/net/html/charset"
 )
 
 var UnknownFormat = errors.New("unknown feed format")
 
-type processor func(r io.Reader) (*Feed, error)
+type feedProbe struct {
+	feedType string
+	callback func(r io.Reader) (*Feed, error)
+	encoding string
+}
 
-func sniff(lookup string) (string, processor) {
+func sniff(lookup string) (out feedProbe) {
 	lookup = strings.TrimSpace(lookup)
 	lookup = strings.TrimLeft(lookup, "\x00\xEF\xBB\xBF\xFE\xFF")
 
-	if len(lookup) < 0 {
-		return "", nil
+	if len(lookup) == 0 {
+		return
 	}
 
 	switch lookup[0] {
@@ -33,24 +38,42 @@ func sniff(lookup string) (string, processor) {
 			if token == nil {
 				break
 			}
+
+			// check <?xml encoding="ENCODING" ?>
+			if el, ok := token.(xml.ProcInst); ok && el.Target == "xml" {
+				out.encoding = strings.ToLower(procInst("encoding", string(el.Inst)))
+			}
+
 			if el, ok := token.(xml.StartElement); ok {
 				switch el.Name.Local {
 				case "rss":
-					return "rss", ParseRSS
+					out.feedType = "rss"
+					out.callback = ParseRSS
+					return
 				case "RDF":
-					return "rdf", ParseRDF
+					out.feedType = "rdf"
+					out.callback = ParseRDF
+					return
 				case "feed":
-					return "atom", ParseAtom
+					out.feedType = "atom"
+					out.callback = ParseAtom
+					return
 				}
 			}
 		}
 	case '{':
-		return "json", ParseJSON
+		out.feedType = "json"
+		out.callback = ParseJSON
+		return
 	}
-	return "", nil
+	return
 }
 
 func Parse(r io.Reader) (*Feed, error) {
+	return ParseWithEncoding(r, "")
+}
+
+func ParseWithEncoding(r io.Reader, fallbackEncoding string) (*Feed, error) {
 	lookup := make([]byte, 2048)
 	n, err := io.ReadFull(r, lookup)
 	switch {
@@ -63,18 +86,42 @@ func Parse(r io.Reader) (*Feed, error) {
 		r = io.MultiReader(bytes.NewReader(lookup), r)
 	}
 
-	_, callback := sniff(string(lookup))
-	if callback == nil {
+	out := sniff(string(lookup))
+	if out.feedType == "" {
 		return nil, UnknownFormat
 	}
 
-	feed, err := callback(r)
+	if out.encoding == "" && fallbackEncoding != "" {
+		r, err = charset.NewReaderLabel(fallbackEncoding, r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if (out.feedType != "json") && (out.encoding == "" || out.encoding == "utf-8") {
+		// XML decoder will not rely on custom CharsetReader (see `xmlDecoder`)
+		// to handle invalid xml characters.
+		// Assume input is already UTF-8 and do the cleanup here.
+		r = NewSafeXMLReader(r)
+	}
+
+	feed, err := out.callback(r)
 	if feed != nil {
 		feed.cleanup()
 	}
 	return feed, err
 }
 
+func ParseAndFix(r io.Reader, baseURL, fallbackEncoding string) (*Feed, error) {
+	feed, err := ParseWithEncoding(r, fallbackEncoding)
+	if err != nil {
+		return nil, err
+	}
+	feed.TranslateURLs(baseURL)
+	feed.SetMissingDatesTo(time.Now())
+	return feed, nil
+}
+
 func (feed *Feed) cleanup() {
 	feed.Title = strings.TrimSpace(feed.Title)
 	feed.SiteURL = strings.TrimSpace(feed.SiteURL)

src/parser/feed_test.go

@@ -7,38 +7,40 @@ import (
 )
 
 func TestSniff(t *testing.T) {
-	testcases := [][2]string{
+	testcases := []struct{
+		input string
+		want feedProbe
+	}{
 		{
 			`<?xml version="1.0"?><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"></rdf:RDF>`,
-			"rdf",
+			feedProbe{feedType: "rdf", callback: ParseRDF},
 		},
 		{
 			`<?xml version="1.0" encoding="ISO-8859-1"?><rss version="2.0"><channel></channel></rss>`,
-			"rss",
+			feedProbe{feedType: "rss", callback: ParseRSS, encoding: "iso-8859-1"},
 		},
 		{
 			`<?xml version="1.0"?><rss version="2.0"><channel></channel></rss>`,
-			"rss",
+			feedProbe{feedType: "rss", callback: ParseRSS},
 		},
 		{
 			`<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>`,
-			"atom",
+			feedProbe{feedType: "atom", callback: ParseAtom, encoding: "utf-8"},
 		},
 		{
 			`{}`,
-			"json",
+			feedProbe{feedType: "json", callback: ParseJSON},
 		},
 		{
 			`<!DOCTYPE html><html><head><title></title></head><body></body></html>`,
-			"",
+			feedProbe{},
 		},
 	}
 	for _, testcase := range testcases {
-		have, _ := sniff(testcase[0])
-		want := testcase[1]
-		if want != have {
-			t.Log(testcase[0])
-			t.Errorf("Invalid format: want=%#v have=%#v", want, have)
+		want := testcase.want
+		have := sniff(testcase.input)
+		if want.encoding != have.encoding || want.feedType != have.feedType {
+			t.Errorf("Invalid output\n---\n%s\n---\n\nwant=%#v\nhave=%#v", testcase.input, want, have)
 		}
 	}
 }
@@ -107,3 +109,44 @@ func TestParseFeedWithBOM(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+func TestParseCleanIllegalCharsInUTF8(t *testing.T) {
+	data := `
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+					<title>` + "\a" + `title</title>
+				</item>
+			</channel>
+		</rss>
+	`
+	feed, err := Parse(strings.NewReader(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(feed.Items) != 1 || feed.Items[0].Title != "title" {
+		t.Fatalf("invalid feed, got: %v", feed)
+	}
+}
+
+func TestParseCleanIllegalCharsInNonUTF8(t *testing.T) {
+	// echo привет | iconv -f utf8 -t cp1251 | hexdump -C
+	data := `
+		<?xml version="1.0" encoding="windows-1251"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+					<title>` + "\a \xef\xf0\xe8\xe2\xe5\xf2\x0a \a" + `</title>
+				</item>
+			</channel>
+		</rss>
+	`
+	feed, err := Parse(strings.NewReader(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(feed.Items) != 1 || feed.Items[0].Title != "привет" {
+		t.Fatalf("invalid feed, got: %v", feed)
+	}
+}

src/parser/rss.go

@@ -22,7 +22,7 @@ type rssFeed struct {
 type rssItem struct {
 	GUID        string         `xml:"guid"`
 	Title       string         `xml:"title"`
-	Link        string         `xml:"link"`
+	Link        string         `xml:"rss link"`
 	Description string         `xml:"rss description"`
 	PubDate     string         `xml:"pubDate"`
 	Enclosures  []rssEnclosure `xml:"enclosure"`

src/parser/util.go

@@ -1,6 +1,8 @@
 package parser
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/xml"
 	"io"
 	"regexp"
@@ -30,6 +32,81 @@ func plain2html(text string) string {
 func xmlDecoder(r io.Reader) *xml.Decoder {
 	decoder := xml.NewDecoder(r)
 	decoder.Strict = false
-	decoder.CharsetReader = charset.NewReaderLabel
+	decoder.CharsetReader = func(cs string, input io.Reader) (io.Reader, error) {
+		r, err := charset.NewReaderLabel(cs, input)
+		if err == nil {
+			r = NewSafeXMLReader(r)
+		}
+		return r, err
+	}
 	return decoder
 }
+
+type safexmlreader struct {
+	reader *bufio.Reader
+	buffer *bytes.Buffer
+}
+
+func NewSafeXMLReader(r io.Reader) io.Reader {
+	return &safexmlreader{
+		reader: bufio.NewReader(r),
+		buffer: bytes.NewBuffer(make([]byte, 0, 4096)),
+	}
+}
+
+func (xr *safexmlreader) Read(p []byte) (int, error) {
+	for xr.buffer.Len() < cap(p) {
+		r, _, err := xr.reader.ReadRune()
+		if err == io.EOF {
+			if xr.buffer.Len() == 0 {
+				return 0, io.EOF
+			}
+			break
+		}
+		if err != nil {
+			return 0, err
+		}
+		if isInCharacterRange(r) {
+			xr.buffer.WriteRune(r)
+		}
+	}
+	return xr.buffer.Read(p)
+}
+
+// NOTE: copied from "encoding/xml" package
+// Decide whether the given rune is in the XML Character Range, per
+// the Char production of https://www.xml.com/axml/testaxml.htm,
+// Section 2.2 Characters.
+func isInCharacterRange(r rune) (inrange bool) {
+	return r == 0x09 ||
+		r == 0x0A ||
+		r == 0x0D ||
+		r >= 0x20 && r <= 0xD7FF ||
+		r >= 0xE000 && r <= 0xFFFD ||
+		r >= 0x10000 && r <= 0x10FFFF
+}
+
+// NOTE: copied from "encoding/xml" package
+// procInst parses the `param="..."` or `param='...'`
+// value out of the provided string, returning "" if not found.
+func procInst(param, s string) string {
+	// TODO: this parsing is somewhat lame and not exact.
+	// It works for all actual cases, though.
+	param = param + "="
+	idx := strings.Index(s, param)
+	if idx == -1 {
+		return ""
+	}
+	v := s[idx+len(param):]
+	if v == "" {
+		return ""
+	}
+	if v[0] != '\'' && v[0] != '"' {
+		return ""
+	}
+	idx = strings.IndexRune(v[1:], rune(v[0]))
+	if idx == -1 {
+		return ""
+	}
+	return v[1 : idx+1]
+}

src/parser/util_test.go

@@ -0,0 +1,88 @@
+package parser
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+)
+
+func TestSafeXMLReader(t *testing.T) {
+	var f io.Reader
+	want := []byte("привет мир")
+	f = bytes.NewReader(want)
+	f = NewSafeXMLReader(f)
+
+	have, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Fatalf("invalid output\nwant: %v\nhave: %v", want, have)
+	}
+}
+
+func TestSafeXMLReaderRemoveUnwantedRunes(t *testing.T) {
+	var f io.Reader
+	input := []byte("\aпривет \x0cмир\ufffe\uffff")
+	want := []byte("привет мир")
+	f = bytes.NewReader(input)
+	f = NewSafeXMLReader(f)
+
+	have, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Fatalf("invalid output\nwant: %v\nhave: %v", want, have)
+	}
+}
+
+func TestSafeXMLReaderPartial1(t *testing.T) {
+	var f io.Reader
+	input := []byte("\aпривет \x0cмир\ufffe\uffff")
+	want := []byte("привет мир")
+	f = bytes.NewReader(input)
+	f = NewSafeXMLReader(f)
+
+	buf := make([]byte, 1)
+	for i := 0; i < len(want); i++ {
+		n, err := f.Read(buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n != 1 {
+			t.Fatalf("expected 1 byte, got %d", n)
+		}
+		if buf[0] != want[i] {
+			t.Fatalf("invalid char at pos %d\nwant: %v\nhave: %v", i, want[i], buf[0])
+		}
+	}
+	if x, err := f.Read(buf); err != io.EOF {
+		t.Fatalf("expected EOF, %v, %v %v", buf, x, err)
+	}
+}
+
+func TestSafeXMLReaderPartial2(t *testing.T) {
+	var f io.Reader
+	input := []byte("привет\a\a\a\a\a")
+	f = bytes.NewReader(input)
+	f = NewSafeXMLReader(f)
+
+	buf := make([]byte, 12)
+	n, err := f.Read(buf)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if n != 12 {
+		t.Fatalf("expected 12 bytes")
+	}
+
+	n, err = f.Read(buf)
+	if n != 0 {
+		t.Fatalf("expected 0")
+	}
+	if err != io.EOF {
+		t.Fatalf("expected EOF, got %v", err)
+	}
+}

src/server/opml/read.go

@@ -3,6 +3,8 @@ package opml
 import (
 	"encoding/xml"
 	"io"
+
+	"golang.org/x/net/html/charset"
 )
 
 type opml struct {
@@ -45,6 +47,7 @@ func Parse(r io.Reader) (Folder, error) {
 	decoder := xml.NewDecoder(r)
 	decoder.Entity = xml.HTMLEntity
 	decoder.Strict = false
+	decoder.CharsetReader = charset.NewReaderLabel
 
 	err := decoder.Decode(&val)
 	if err != nil {

src/server/opml/read_test.go

@@ -1,6 +1,7 @@
 package opml
 
 import (
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -87,3 +88,41 @@ func TestParseFallback(t *testing.T) {
 		t.Fatal("invalid opml")
 	}
 }
+
+func TestParseWithEncoding(t *testing.T) {
+	file, err := os.Open("sample_win1251.xml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	have, err := Parse(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := Folder{
+		Title: "",
+		Feeds: []Feed{
+			{
+				Title:   "пример1",
+				FeedUrl: "https://baz.com/feed.xml",
+				SiteUrl: "https://baz.com/",
+			},
+		},
+		Folders: []Folder{
+			{
+				Title: "папка",
+				Feeds: []Feed{
+					{
+						Title:   "пример2",
+						FeedUrl: "https://foo.com/feed.xml",
+						SiteUrl: "https://foo.com/",
+					},
+				},
+			},
+		},
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid opml")
+	}
+}

src/server/opml/sample_win1251.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="windows-1251"?>
+<opml version="1.1">
+<head><title><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD></title></head>
+<body>
+    <outline text="<22><><EFBFBD><EFBFBD><EFBFBD>">
+        <outline type="rss" text="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2" description="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2" xmlUrl="https://foo.com/feed.xml" htmlUrl="https://foo.com/"/>
+    </outline>
+    <outline type="rss" text="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1" description="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1" xmlUrl="https://baz.com/feed.xml" htmlUrl="https://baz.com/"/>
+</body>
+</opml>

src/server/routes.go

@@ -219,7 +219,11 @@ func (s *Server) handleFeedList(c *router.Context) {
 				result.FeedLink,
 				form.FolderID,
 			)
-			s.db.CreateItems(worker.ConvertItems(result.Feed.Items, *feed))
+			items := worker.ConvertItems(result.Feed.Items, *feed)
+			if len(items) > 0 {
+				s.db.CreateItems(items)
+				s.db.SetFeedSize(feed.Id, len(items))
+			}
 			s.worker.FindFeedFavicon(*feed)
 
 			c.JSON(http.StatusOK, map[string]interface{}{
@@ -457,14 +461,13 @@ func (s *Server) handlePageCrawl(c *router.Context) {
 		return
 	}
 
-	res, err := http.Get(url)
+	body, err := worker.GetBody(url)
 	if err != nil {
 		log.Print(err)
 		c.Out.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	defer res.Body.Close()
-	content, err := readability.ExtractContent(res.Body)
+	content, err := readability.ExtractContent(strings.NewReader(body))
 	if err != nil {
 		log.Print(err)
 		c.Out.WriteHeader(http.StatusNoContent)

src/storage/feed.go

@@ -194,3 +194,15 @@ func (s *Storage) GetFeedErrors() map[int64]string {
 	}
 	return errors
 }
+
+func (s *Storage) SetFeedSize(feedId int64, size int) {
+	_, err := s.db.Exec(`
+		insert into feed_sizes (feed_id, size)
+		values (?, ?)
+		on conflict (feed_id) do update set size = excluded.size`,
+		feedId, size,
+	)
+	if err != nil {
+		log.Print(err)
+	}
+}

src/storage/item.go

@@ -61,7 +61,7 @@ type ItemFilter struct {
 	FeedID   *int64
 	Status   *ItemStatus
 	Search   *string
-	After  *int64
+	After    *int64
 }
 
 type MarkFilter struct {
@@ -292,45 +292,70 @@ func (s *Storage) SyncSearch() {
 	}
 }
 
+var (
+	itemsKeepSize = 50
+	itemsKeepDays = 90
+)
+
+// Delete old articles from the database to cleanup space.
+//
+// The rules:
+// * Never delete starred entries.
+// * Keep at least the same amount of articles the feed provides (default: 50).
+//   This prevents from deleting items for rarely updated and/or ever-growing
+//   feeds which might eventually reappear as unread.
+// * Keep entries for a certain period (default: 90 days).
 func (s *Storage) DeleteOldItems() {
-	rows, err := s.db.Query(fmt.Sprintf(`
-		select feed_id, count(*) as num_items
-		from items
-		where status != %d
-		group by feed_id
-		having num_items > 50
-	`, STARRED))
+	rows, err := s.db.Query(`
+		select
+			i.feed_id,
+			max(coalesce(s.size, 0), ?) as max_items,
+			count(*) as num_items
+		from items i
+		left outer join feed_sizes s on s.feed_id = i.feed_id
+		where status != ?
+		group by i.feed_id
+	`, itemsKeepSize, STARRED)
 
 	if err != nil {
 		log.Print(err)
 		return
 	}
 
-	feedIds := make([]int64, 0)
+	feedLimits := make(map[int64]int64, 0)
 	for rows.Next() {
-		var id int64
-		rows.Scan(&id, nil)
-		feedIds = append(feedIds, id)
+		var feedId, limit int64
+		rows.Scan(&feedId, &limit, nil)
+		feedLimits[feedId] = limit
 	}
 
-	for _, feedId := range feedIds {
+	for feedId, limit := range feedLimits {
 		result, err := s.db.Exec(`
-			delete from items where feed_id = ? and status != ? and date_arrived < ?`,
+			delete from items
+			where id in (
+				select i.id
+				from items i
+				where i.feed_id = ? and status != ?
+				order by date desc
+				limit -1 offset ?
+			) and date_arrived < ?
+			`,
 			feedId,
 			STARRED,
-			time.Now().Add(-time.Hour*24*90), // 90 days
+			limit,
+			time.Now().Add(-time.Hour*time.Duration(24*itemsKeepDays)),
 		)
 		if err != nil {
 			log.Print(err)
 			return
 		}
-		num, err := result.RowsAffected()
+		numDeleted, err := result.RowsAffected()
 		if err != nil {
 			log.Print(err)
 			return
 		}
-		if num > 0 {
-			log.Printf("Deleted %d old items (%d)", num, feedId)
+		if numDeleted > 0 {
+			log.Printf("Deleted %d old items (feed: %d)", numDeleted, feedId)
 		}
 	}
 }

src/storage/item_test.go

@@ -3,6 +3,7 @@ package storage
 import (
 	"log"
 	"reflect"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -45,14 +46,14 @@ func testItemsSetup(db *Storage) testItemScope {
 	db.CreateItems([]Item{
 		// feed11
 		{GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)},
-		{GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)},  // read
-		{GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)},  // starred
+		{GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)}, // read
+		{GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)}, // starred
 		// feed12
 		{GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)},
-		{GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)},  // read
+		{GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)}, // read
 		// feed21
-		{GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)},  // read
-		{GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)},  // starred
+		{GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)}, // read
+		{GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)}, // starred
 		// feed01
 		{GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)},
 		{GUID: "item012", FeedId: feed01.Id, Title: "title012", Date: now.Add(time.Hour * 24 * 9)},  // read
@@ -271,3 +272,59 @@ func TestMarkItemsRead(t *testing.T) {
 		t.Fail()
 	}
 }
+
+func TestDeleteOldItems(t *testing.T) {
+	extraItems := 10
+
+	now := time.Now()
+	db := testDB()
+	feed := db.CreateFeed("feed", "", "", "http://test.com/feed11.xml", nil)
+
+	items := make([]Item, 0)
+	for i := 0; i < itemsKeepSize+extraItems; i++ {
+		istr := strconv.Itoa(i)
+		items = append(items, Item{
+			GUID:   istr,
+			FeedId: feed.Id,
+			Title:  istr,
+			Date:   now.Add(time.Hour * time.Duration(i)),
+		})
+	}
+	db.CreateItems(items)
+	
+	db.SetFeedSize(feed.Id, itemsKeepSize)
+	var feedSize int
+	err := db.db.QueryRow(
+		`select size from feed_sizes where feed_id = ?`, feed.Id,
+	).Scan(&feedSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if feedSize != itemsKeepSize {
+		t.Fatalf(
+			"expected feed size to get updated\nwant: %d\nhave: %d", 
+			itemsKeepSize+extraItems,
+			feedSize,
+		)
+	}
+
+	// expire only the first 3 articles
+	_, err = db.db.Exec(
+		`update items set date_arrived = ?
+		where id in (select id from items limit 3)`,
+		now.Add(-time.Hour*time.Duration(itemsKeepDays*24)),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db.DeleteOldItems()
+	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false)
+	if len(feedItems) != len(items)-3 {
+		t.Fatalf(
+			"invalid number of old items kept\nwant: %d\nhave: %d",
+			len(items)-3,
+			len(feedItems),
+		)
+	}
+}

src/storage/migration.go

@@ -13,6 +13,7 @@ var migrations = []func(*sql.Tx) error{
 	m04_item_podcasturl,
 	m05_move_description_to_content,
 	m06_fill_missing_dates,
+	m07_add_feed_size,
 }
 
 var maxVersion = int64(len(migrations))
@@ -259,3 +260,14 @@ func m06_fill_missing_dates(tx *sql.Tx) error {
 	_, err := tx.Exec(sql)
 	return err
 }
+
+func m07_add_feed_size(tx *sql.Tx) error {
+	sql := `
+		create table if not exists feed_sizes (
+		 feed_id        references feeds(id) on delete cascade unique,
+		 size           integer not null default 0
+		);
+	`
+	_, err := tx.Exec(sql)
+	return err
+}

src/storage/storage_test.go

@@ -11,6 +11,7 @@ func testDB() *Storage {
 	log.SetOutput(io.Discard)
 	db, _ := New(":memory:")
 	log.SetOutput(os.Stderr)
+	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
 	return db
 }
 

src/systray/NOTES.txt

@@ -9,3 +9,4 @@ hash:
 changes:
 
 -removed `getlantern/golog` dependency
+-prevent from compiling in linux

src/systray/systray.go

@@ -1,3 +1,5 @@
+// +build darwin windows
+
 /*
 Package systray is a cross-platform Go library to place an icon and menu in the notification area.
 */

src/systray/systray_linux.go

@@ -1,3 +1,5 @@
+// +build never
+
 package systray
 
 /*

src/systray/systray_nonwindows.go

@@ -1,9 +1,8 @@
-// +build !windows
+// +build darwin
 
 package systray
 
 /*
-#cgo linux pkg-config: gtk+-3.0 appindicator3-0.1
 #cgo darwin CFLAGS: -DDARWIN -x objective-c -fobjc-arc
 #cgo darwin LDFLAGS: -framework Cocoa
 

src/worker/crawler.go

@@ -6,10 +6,10 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime"
 	"net/http"
 	"net/url"
 	"strings"
-	"time"
 
 	"github.com/nkanaev/yarr/src/content/scraper"
 	"github.com/nkanaev/yarr/src/parser"
@@ -39,29 +39,32 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
 	if res.StatusCode != 200 {
 		return nil, fmt.Errorf("status code %d", res.StatusCode)
 	}
+	cs := getCharset(res)
 
-	body, err := httpBody(res)
-	if err != nil {
-		return nil, err
-	}
-	content, err := ioutil.ReadAll(body)
+	body, err := io.ReadAll(res.Body)
 	if err != nil {
 		return nil, err
 	}
 
 	// Try to feed into parser
-	feed, err := parser.Parse(bytes.NewReader(content))
+	feed, err := parser.ParseAndFix(bytes.NewReader(body), candidateUrl, cs)
 	if err == nil {
-		feed.TranslateURLs(candidateUrl)
-		feed.SetMissingDatesTo(time.Now())
 		result.Feed = feed
 		result.FeedLink = candidateUrl
 		return result, nil
 	}
 
 	// Possibly an html link. Search for feed links
+	content := string(body)
+	if cs != "" {
+		if r, err := charset.NewReaderLabel(cs, bytes.NewReader(body)); err == nil {
+			if body, err := io.ReadAll(r); err == nil {
+				content = string(body)
+			}
+		}
+	}
 	sources := make([]FeedSource, 0)
-	for url, title := range scraper.FindFeeds(string(content), candidateUrl) {
+	for url, title := range scraper.FindFeeds(content, candidateUrl) {
 		sources = append(sources, FeedSource{Title: title, Url: url})
 	}
 	switch {
@@ -187,12 +190,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 		return nil, nil
 	}
 
-	body, err := httpBody(res)
-	if err != nil {
-		return nil, err
-	}
-
-	feed, err := parser.Parse(body)
+	feed, err := parser.ParseAndFix(res.Body, f.FeedLink, getCharset(res))
 	if err != nil {
 		return nil, err
 	}
@@ -202,15 +200,42 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 	if lmod != "" || etag != "" {
 		db.SetHTTPState(f.Id, lmod, etag)
 	}
-	feed.TranslateURLs(f.FeedLink)
-	feed.SetMissingDatesTo(time.Now())
 	return ConvertItems(feed.Items, f), nil
 }
 
-func httpBody(res *http.Response) (io.Reader, error) {
+func getCharset(res *http.Response) string {
+	contentType := res.Header.Get("Content-Type")
+	if _, params, err := mime.ParseMediaType(contentType); err == nil {
+		if cs, ok := params["charset"]; ok {
+			if e, _ := charset.Lookup(cs); e != nil {
+				return cs
+			}
+		}
+	}
+	return ""
+}
+
+func GetBody(url string) (string, error) {
+	res, err := client.get(url)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+
+	var r io.Reader
+
 	ctype := res.Header.Get("Content-Type")
 	if strings.Contains(ctype, "charset") {
-		return charset.NewReader(res.Body, ctype)
+		r, err = charset.NewReader(res.Body, ctype)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		r = res.Body
 	}
-	return res.Body, nil
+	body, err := io.ReadAll(r)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
 }

src/worker/worker.go

@@ -121,7 +121,11 @@ func (w *Worker) refresher(feeds []storage.Feed) {
 		srcqueue <- feed
 	}
 	for i := 0; i < len(feeds); i++ {
-		w.db.CreateItems(<-dstqueue)
+		items := <-dstqueue
+		if len(items) > 0 {
+			w.db.CreateItems(items)
+			w.db.SetFeedSize(items[0].FeedId, len(items))
+		}
 		atomic.AddInt32(w.pending, -1)
 		w.db.SyncSearch()
 	}