v2.2

This patch makes categorising new feeds a bit more intuitive:
the selected folder (or feed within a folder) in the feed list
will automatically be selected when adding a new feed.

.github/workflows/build.yml

@@ -131,7 +131,7 @@ jobs:
       with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
           asset_path: ./yarr-windows.zip
-          asset_name: yarr-${{ github.ref }}-windows32.zip
+          asset_name: yarr-${{ github.ref }}-windows64.zip
           asset_content_type: application/zip
     - name: Upload Linux
       uses: actions/upload-release-asset@v1
@@ -140,5 +140,5 @@ jobs:
       with:
           upload_url: ${{ steps.create_release.outputs.upload_url }}
           asset_path: ./yarr-linux.zip
-          asset_name: yarr-${{ github.ref }}-linux32.zip
+          asset_name: yarr-${{ github.ref }}-linux64.zip
           asset_content_type: application/zip

.gitignore

@@ -1,5 +1,3 @@
-/server/assets.go
-/gofeed
 /_output
 /yarr
 *.db

doc/changelog.txt

@@ -1,3 +1,21 @@
+# upcoming
+
+- (fix) windows console support (thanks to @dufferzafar for the report)
+- (fix) remove html tags from article titles (thanks to Alex Went for the report)
+- (etc) autoselect current folder when adding a new feed (thanks to @krkk)
+- (etc) folder/feed settings menu available across all filters
+
+# v2.1 (2021-08-16)
+
+- (new) configuration via env variables
+- (fix) missing `content-type` headers (thanks to @verahawk for the report)
+- (fix) handle opml files not following the spec (thanks to @huangnauh for the report)
+- (fix) pagination in unread/starred feeds (thanks to @Farow for the report)
+- (fix) handling feeds with non-utf8 encodings (thanks to @fserb for the report)
+- (fix) errors caused by empty feeds (thanks to @decke)
+- (fix) recognize all audio mime types as podcasts (thanks to @krkk)
+- (fix) ui tweaks (thanks to @Farow)
+
 # v2.0 (2021-04-18)
 
 - (new) user interface tweaks

doc/formats.txt

@@ -159,6 +159,7 @@ Delete any from the list in case they drop support of web feeds.
 - medium
 - posthaven
 - reddit
+- substack
 - tumblr
 - vimeo
 - wordpress

doc/thirdparty.txt

@@ -20,3 +20,8 @@ The licenses are included, and the authorship comments are left intact.
   https://github.com/getlantern/systray (commit:2c0986d) Apache 2.0
 
   removed golog dependency
+
+- fixconsole
+  https://github.com/apenwarr/fixconsole (commit:5a9f648) Apache 2.0
+
+  removed `w32` dependency

doc/todo.txt

@@ -0,0 +1 @@
+- feedlist keyboard navigation is flaky in "unread" section

go.mod

@@ -3,7 +3,7 @@ module github.com/nkanaev/yarr
 go 1.16
 
 require (
-	github.com/mattn/go-sqlite3 v1.14.0
-	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
-	golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13
+	github.com/mattn/go-sqlite3 v1.14.7
+	golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420
+	golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6
 )

go.sum

@@ -1,15 +1,12 @@
-github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
-github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
-github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13 h1:5jaG59Zhd+8ZXe8C+lgiAGqkOaZBruqrWclLkgAww34=
-golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
+github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 h1:a8jGStKg0XqKDlKqjLrXn0ioF5MH36pT7Z0BRTqLhbk=
+golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6 h1:cdsMqa2nXzqlgs183pHxtvoVwU7CyzaCTAUOg94af4c=
+golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

makefile

@@ -1,4 +1,4 @@
-VERSION=2.0
+VERSION=2.2
 GITHASH=$(shell git rev-parse --short=8 HEAD)
 
 CGO_ENABLED=1
@@ -11,26 +11,20 @@ build_default:
 	go build -tags "sqlite_foreign_keys release" -ldflags="$(GO_LDFLAGS)" -o _output/yarr src/main.go
 
 build_macos:
-	set GOOS=darwin
-	set GOARCH=amd64
 	mkdir -p _output/macos
-	go build -tags "sqlite_foreign_keys release macos" -ldflags="$(GO_LDFLAGS)" -o _output/macos/yarr src/main.go
+	GOOS=darwin GOARCH=amd64 go build -tags "sqlite_foreign_keys release macos" -ldflags="$(GO_LDFLAGS)" -o _output/macos/yarr src/main.go
 	cp src/platform/icon.png _output/macos/icon.png
 	go run bin/package_macos.go -outdir _output/macos -version "$(VERSION)"
 
 build_linux:
-	set GOOS=linux
-	set GOARCH=386
 	mkdir -p _output/linux
-	go build -tags "sqlite_foreign_keys release linux" -ldflags="$(GO_LDFLAGS)" -o _output/linux/yarr src/main.go
+	GOOS=linux GOARCH=amd64 go build -tags "sqlite_foreign_keys release linux" -ldflags="$(GO_LDFLAGS)" -o _output/linux/yarr src/main.go
 
 build_windows:
-	set GOOS=windows
-	set GOARCH=386
 	mkdir -p _output/windows
 	go run bin/generate_versioninfo.go -version "$(VERSION)" -outfile src/platform/versioninfo.rc
 	windres -i src/platform/versioninfo.rc -O coff -o src/platform/versioninfo.syso
-	go build -tags "sqlite_foreign_keys release windows" -ldflags="$(GO_LDFLAGS) -H windowsgui" -o _output/windows/yarr.exe src/main.go
+	GOOS=windows GOARCH=amd64 go build -tags "sqlite_foreign_keys release windows" -ldflags="$(GO_LDFLAGS) -H windowsgui" -o _output/windows/yarr.exe src/main.go
 
 serve:
 	go run -tags "sqlite_foreign_keys" src/main.go -db local.db

readme.md

@@ -15,20 +15,15 @@ The latest prebuilt binaries for Linux/MacOS/Windows are available
 ### macos
 
 Download `yarr-*-macos64.zip`, unzip it, place `yarr.app` in `/Applications` folder.
-
-The binaries are not signed, because the author doesn't want to buy a certificate.
-Apple hates cheapskate developers, therefore the OS will refuse to run the application.
-To bypass these measures, you can run the command:
+To open the app follow the instructions provided [here][macos-open] or run the command below:
 
     xattr -d com.apple.quarantine /Applications/yarr.app
 
+[macos-open]: https://support.apple.com/en-gb/guide/mac-help/mh40616/mac
+
 ### windows
 
-Download `yarr-*-windows32.zip`, unzip it, place wherever you'd like to
-(`C:\Program Files` or Recycle Bin). Create a shortcut manually if you'd like to.
-
-Microsoft doesn't like cheapskate developers too,
-but might only gently warn you about that, which you can safely ignore.
+Download `yarr-*-windows32.zip`, unzip it, open `yarr.exe`
 
 ### linux
 

src/assets/index.html

@@ -57,6 +57,18 @@
 
                     <div class="dropdown-divider"></div>
 
+                    <header class="dropdown-header">Theme</header>
+                    <div class="row text-center m-0">
+                        <button class="btn btn-link col-4 px-0 rounded-0"
+                                :class="'theme-'+t"
+                                @click.stop="theme.name = t"
+                                v-for="t in ['light', 'sepia', 'night']">
+                            <span class="icon" v-if="theme.name == t">{% inline "check.svg" %}</span>
+                        </button>
+                    </div>
+
+                    <div class="dropdown-divider"></div>
+
                     <header class="dropdown-header">Auto Refresh</header>
                     <div class="row text-center m-0">
                         <button class="dropdown-item col-4 px-0" :class="{active: !refreshRate}" @click.stop="refreshRate = 0">0</button>
@@ -117,6 +129,7 @@
                 <div v-for="folder in foldersWithFeeds">
                     <label class="selectgroup mt-1"
                            :class="{'d-none': filterSelected
+                                              && !(current.folder.id == folder.id || current.feed.folder_id == folder.id)
                                               && !filteredFolderStats[folder.id]
                                               && (!itemSelectedDetails || feedsById[itemSelectedDetails.feed_id].folder_id != folder.id)}">
                         <input type="radio" name="feed" :value="'folder:'+folder.id" v-model="feedSelected">
@@ -133,6 +146,7 @@
                     <div v-show="!folder.id || folder.is_expanded" class="mt-1" :class="{'pl-3': folder.id}">
                         <label class="selectgroup"
                               :class="{'d-none': filterSelected
+                                                 && !(current.feed.id == feed.id)
                                                  && !filteredFeedStats[feed.id]
                                                  && (!itemSelectedDetails || itemSelectedDetails.feed_id != feed.id)}"
                                v-for="feed in folder.feeds">
@@ -177,11 +191,16 @@
                         title="Mark All Read">
                     <span class="icon">{% inline "check.svg" %}</span>
                 </button>
+
+
+                <button class="btn btn-link toolbar-item px-2 ml-2" v-if="!current.type" disabled>
+                    <span class="icon">{% inline "more-horizontal.svg" %}</span>
+                </button>
                 <dropdown class="settings-dropdown"
                           toggle-class="btn btn-link toolbar-item px-2 ml-2"
                           drop="right"
                           title="Feed Settings"
-                          v-if="!filterSelected && current.type == 'feed'">
+                          v-if="current.type == 'feed'">
                     <template v-slot:button>
                         <span class="icon">{% inline "more-horizontal.svg" %}</span>
                     </template>
@@ -226,7 +245,7 @@
                           toggle-class="btn btn-link toolbar-item px-2 ml-2"
                           title="Folder Settings"
                           drop="right"
-                          v-if="!filterSelected && current.type == 'folder'">
+                          v-if="current.type == 'folder'">
                     <template v-slot:button>
                         <span class="icon">{% inline "more-horizontal.svg" %}</span>
                     </template>
@@ -255,12 +274,12 @@
                             <small class="flex-fill text-truncate mr-1">
                                 {{ feedsById[item.feed_id].title }}
                             </small>
-                            <small class="flex-shrink-0"><relative-time :val="item.date"/></small>
+                            <small class="flex-shrink-0"><relative-time v-bind:title="formatDate(item.date)" :val="item.date"/></small>
                         </div>
                         <div>{{ item.title || 'untitled' }}</div>
                     </div>
                 </label>
-                <button class="btn btn-link btn-block loading my-3" v-if="itemsPage.cur < itemsPage.num"></button>
+                <button class="btn btn-link btn-block loading my-3" v-if="itemsHasMore"></button>
             </div>
             <div class="px-3 py-2 border-top text-danger text-break" v-if="feed_errors[current.feed.id]">
                 {{ feed_errors[current.feed.id] }}
@@ -285,14 +304,6 @@
                     <template v-slot:button>
                         <span class="icon">{% inline "sliders.svg" %}</span>
                     </template>
-                    <div class="row text-center m-0">
-                        <button class="btn btn-link col-4 px-0 rounded-0"
-                                :class="'theme-'+t"
-                                @click.stop="theme.name = t"
-                                v-for="t in ['light', 'sepia', 'night']">
-                            <span class="icon" v-if="theme.name == t">{% inline "check.svg" %}</span>
-                        </button>
-                    </div>
 
                     <button class="dropdown-item" :class="{active: !theme.font}" @click.stop="theme.font = ''">sans-serif</button>
                     <button class="dropdown-item font-serif" :class="{active: theme.font == 'serif'}" @click.stop="theme.font = 'serif'">serif</button>
@@ -350,7 +361,7 @@
                     </label>
                     <select class="form-control" id="feed-folder" name="folder_id" ref="newFeedFolder">
                         <option value="">---</option>
-                        <option :value="folder.id" v-for="folder in folders">{{ folder.title }}</option>
+                        <option :value="folder.id" v-for="folder in folders" :selected="folder.id === current.feed.folder_id || folder.id === current.folder.id">{{ folder.title }}</option>
                     </select>
                     <div class="mt-4" v-if="feedNewChoice.length">
                         <p class="mb-2">

src/assets/javascripts/app.js

@@ -47,13 +47,13 @@ Vue.component('drag', {
 })
 
 Vue.component('dropdown', {
-  props: ['class', 'toggle-class', 'ref', 'drop'],
+  props: ['class', 'toggle-class', 'ref', 'drop', 'title'],
   data: function() {
     return {open: false}
   },
   template: `
     <div class="dropdown" :class="$attrs.class">
-      <button ref="btn" @click="toggle" :class="btnToggleClass"><slot name="button"></slot></button>
+      <button ref="btn" @click="toggle" :class="btnToggleClass" :title="$props.title"><slot name="button"></slot></button>
       <div ref="menu" class="dropdown-menu" :class="{show: open}"><slot v-if="open"></slot></div>
     </div>
   `,
@@ -180,7 +180,7 @@ var vm = new Vue({
   created: function() {
     this.refreshStats()
       .then(this.refreshFeeds.bind(this))
-      .then(this.refreshItems.bind(this))
+      .then(this.refreshItems.bind(this, false))
 
     api.feeds.list_errors().then(function(errors) {
       vm.feed_errors = errors
@@ -197,10 +197,7 @@ var vm = new Vue({
       'feedNewChoice': [],
       'feedNewChoiceSelected': '',
       'items': [],
-      'itemsPage': {
-        'cur': 1,
-        'num': 1,
-      },
+      'itemsHasMore': true,
       'itemSelected': null,
       'itemSelectedDetails': null,
       'itemSelectedReadability': '',
@@ -304,13 +301,13 @@ var vm = new Vue({
     },
     'filterSelected': function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
-      api.settings.update({filter: newVal}).then(this.refreshItems.bind(this))
+      api.settings.update({filter: newVal}).then(this.refreshItems.bind(this, false))
       this.itemSelected = null
       this.computeStats()
     },
     'feedSelected': function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
-      api.settings.update({feed: newVal}).then(this.refreshItems.bind(this))
+      api.settings.update({feed: newVal}).then(this.refreshItems.bind(this, false))
       this.itemSelected = null
       if (this.$refs.itemlist) this.$refs.itemlist.scrollTop = 0
     },
@@ -339,7 +336,7 @@ var vm = new Vue({
     }, 500),
     'itemSortNewestFirst': function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
-      api.settings.update({sort_newest_first: newVal}).then(this.refreshItems.bind(this))
+      api.settings.update({sort_newest_first: newVal}).then(vm.refreshItems.bind(this, false))
     },
     'feedListWidth': debounce(function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
@@ -404,34 +401,34 @@ var vm = new Vue({
           vm.feeds = values[1]
         })
     },
-    refreshItems: function() {
+    refreshItems: function(loadMore) {
       if (this.feedSelected === null) {
         vm.items = []
-        vm.itemsPage = {'cur': 1, 'num': 1}
         return
       }
+
       var query = this.getItemsQuery()
+      if (loadMore) {
+        query.after = vm.items[vm.items.length-1].id
+      }
+
       this.loading.items = true
       return api.items.list(query).then(function(data) {
-        vm.items = data.list
-        vm.itemsPage = data.page
+        if (loadMore) {
+          vm.items = vm.items.concat(data.list)
+        } else {
+          vm.items = data.list
+        }
+        vm.itemsHasMore = data.has_more
         vm.loading.items = false
       })
     },
     loadMoreItems: function(event, el) {
-      if (this.itemsPage.cur >= this.itemsPage.num) return
+      if (!this.itemsHasMore) return
+
       if (this.loading.items) return
       var closeToBottom = (el.scrollHeight - el.scrollTop - el.offsetHeight) < 50
-      if (closeToBottom) {
-        this.loading.moreitems = true
-        var query = this.getItemsQuery()
-        query.page = this.itemsPage.cur + 1
-        api.items.list(query).then(function(data) {
-          vm.items = vm.items.concat(data.list)
-          vm.itemsPage = data.page
-          vm.loading.items = false
-        })
-      }
+      if (closeToBottom) this.refreshItems(true)
     },
     markItemsRead: function() {
       var query = this.getItemsQuery()
@@ -439,6 +436,7 @@ var vm = new Vue({
         vm.items = []
         vm.itemsPage = {'cur': 1, 'num': 1}
         vm.itemSelected = null
+        vm.itemsHasMore = false
         vm.refreshStats()
       })
     },
@@ -633,10 +631,7 @@ var vm = new Vue({
     fetchAllFeeds: function() {
       if (this.loading.feeds) return
       api.feeds.refresh().then(function() {
-        // NOTE: this is hacky
-        setTimeout(function() {
-          vm.refreshStats()
-        }, 1000)
+        vm.refreshStats()
       })
     },
     computeStats: function() {

src/assets/login.html

@@ -30,7 +30,7 @@
         <div class="form-group">
             <label for="username">Username</label>
             <input name="username" class="form-control" id="username" autocomplete="off"
-                   value="{% if .username %}{% .username %}{% end %}" required>
+                   value="{% if .username %}{% .username %}{% end %}" required autofocus>
         </div>
         <div class="form-group">
             <label for="password">Password</label>

src/assets/stylesheets/app.css

@@ -360,6 +360,27 @@ select.form-control:not([multiple]):not([size]) {
   margin-bottom: 0.5rem;
 }
 
+.content .video-wrapper {
+  position: relative;
+  display: block;
+  width: 100%;
+  overflow: hidden;
+}
+
+.content .video-wrapper::before {
+  display: block;
+  padding-top: 56.25%; /* 16x9 aspect ratio */
+  content: "";
+}
+
+.content .video-wrapper iframe {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+}
+
 .content pre {
   overflow-x: auto;
   color: inherit;

src/content/sanitizer/sanitizer.go

@@ -58,19 +58,36 @@ func Sanitize(baseURL, input string) string {
 				attrNames, htmlAttributes := sanitizeAttributes(baseURL, tagName, token.Attr)
 
 				if hasRequiredAttributes(tagName, attrNames) {
+					wrap := isVideoIframe(token)
+					if wrap {
+						buffer.WriteString(`<div class="video-wrapper">`)
+					}
+
 					if len(attrNames) > 0 {
 						buffer.WriteString("<" + tagName + " " + htmlAttributes + ">")
 					} else {
 						buffer.WriteString("<" + tagName + ">")
 					}
 
-					tagStack = append(tagStack, tagName)
+					if tagName == "iframe" {
+						// autoclose iframes
+						buffer.WriteString("</iframe>")
+						if wrap {
+							buffer.WriteString("</div>")
+						}
+					} else {
+						tagStack = append(tagStack, tagName)
+					}
 				}
 			} else if isBlockedTag(tagName) {
 				blacklistedTagDepth++
 			}
 		case html.EndTagToken:
 			tagName := token.Data
+			// iframes are autoclosed. see above
+			if tagName == "iframe" {
+				continue
+			}
 			if isValidTag(tagName) && inList(tagName, tagStack) {
 				buffer.WriteString(fmt.Sprintf("</%s>", tagName))
 			} else if isBlockedTag(tagName) {
@@ -417,3 +434,22 @@ func isValidDataAttribute(value string) bool {
 	}
 	return false
 }
+
+func isVideoIframe(token html.Token) bool {
+	videoWhitelist := map[string]bool{
+		"player.bilibili.com":      true,
+		"player.vimeo.com":         true,
+		"www.dailymotion.com":      true,
+		"www.youtube-nocookie.com": true,
+		"www.youtube.com":          true,
+	}
+	if token.Data == "iframe" {
+		for _, attr := range token.Attr {
+			if attr.Key == "src" {
+				domain := htmlutil.URLDomain(attr.Val)
+				return videoWhitelist[domain]
+			}
+		}
+	}
+	return false
+}

src/content/sanitizer/sanitizer_test.go

@@ -163,6 +163,16 @@ func TestInvalidNestedTag(t *testing.T) {
 	}
 }
 
+func TestValidIFrame(t *testing.T) {
+	input := `<iframe src="http://example.org/"></iframe>`
+	expected := `<iframe src="http://example.org/" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf("Wrong output:\nwant: %s\nhave: %s", expected, output)
+	}
+}
+
 func TestInvalidIFrame(t *testing.T) {
 	input := `<iframe src="http://example.org/"></iframe>`
 	expected := ``
@@ -175,7 +185,7 @@ func TestInvalidIFrame(t *testing.T) {
 
 func TestIFrameWithChildElements(t *testing.T) {
 	input := `<iframe src="https://www.youtube.com/"><p>test</p></iframe>`
-	expected := `<iframe src="https://www.youtube.com/" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	expected := `<div class="video-wrapper"><iframe src="https://www.youtube.com/" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe></div>`
 	output := Sanitize("http://example.com/", input)
 
 	if expected != output {
@@ -255,7 +265,7 @@ func TestEspaceAttributes(t *testing.T) {
 
 func TestReplaceIframeURL(t *testing.T) {
 	input := `<iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0"></iframe>`
-	expected := `<iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	expected := `<div class="video-wrapper"><iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe></div>`
 	output := Sanitize("http://example.org/", input)
 
 	if expected != output {
@@ -292,3 +302,13 @@ func TestReplaceStyle(t *testing.T) {
 		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
 	}
 }
+
+func TestWrapYoutubeIFrames(t *testing.T) {
+	input := `<iframe src="https://www.youtube.com/embed/foobar"></iframe>`
+	expected := `<div class="video-wrapper"><iframe src="https://www.youtube.com/embed/foobar" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe></div>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf("Wrong output:\nwant: %v\nhave: %v", expected, output)
+	}
+}

src/main.go

@@ -17,18 +17,40 @@ import (
 var Version string = "0.0"
 var GitHash string = "unknown"
 
-func main() {
-	log.SetOutput(os.Stdout)
-	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
+var OptList = make([]string, 0)
 
-	var addr, db, authfile, certfile, keyfile, basepath string
+func opt(envVar, defaultValue string) string {
+	OptList = append(OptList, envVar)
+	value := os.Getenv(envVar)
+	if value != "" {
+		return value
+	}
+	return defaultValue
+}
+
+func main() {
+	platform.FixConsoleIfNeeded()
+
+	var addr, db, authfile, certfile, keyfile, basepath, logfile string
 	var ver, open bool
-	flag.StringVar(&addr, "addr", "127.0.0.1:7070", "address to run server on")
-	flag.StringVar(&authfile, "auth-file", "", "path to a file containing username:password")
-	flag.StringVar(&basepath, "base", "", "base path of the service url")
-	flag.StringVar(&certfile, "cert-file", "", "path to cert file for https")
-	flag.StringVar(&keyfile, "key-file", "", "path to key file for https")
-	flag.StringVar(&db, "db", "", "storage file path")
+
+	flag.CommandLine.SetOutput(os.Stdout)
+
+	flag.Usage = func() {
+		out := flag.CommandLine.Output()
+		fmt.Fprintf(out, "Usage of %s:\n", os.Args[0])
+		flag.PrintDefaults()
+		fmt.Fprintln(out, "\nThe environmental variables, if present, will be used to provide\nthe default values for the params above:")
+		fmt.Fprintln(out, " ", strings.Join(OptList, ", "))
+	}
+
+	flag.StringVar(&addr, "addr", opt("YARR_ADDR", "127.0.0.1:7070"), "address to run server on")
+	flag.StringVar(&basepath, "base", opt("YARR_BASE", ""), "base path of the service url")
+	flag.StringVar(&authfile, "auth-file", opt("YARR_AUTHFILE", ""), "`path` to a file containing username:password")
+	flag.StringVar(&certfile, "cert-file", opt("YARR_CERTFILE", ""), "`path` to cert file for https")
+	flag.StringVar(&keyfile, "key-file", opt("YARR_KEYFILE", ""), "`path` to key file for https")
+	flag.StringVar(&db, "db", opt("YARR_DB", ""), "storage file `path`")
+	flag.StringVar(&logfile, "log-file", opt("YARR_LOGFILE", ""), "`path` to log file to use instead of stdout")
 	flag.BoolVar(&ver, "version", false, "print application version")
 	flag.BoolVar(&open, "open", false, "open the server in browser")
 	flag.Parse()
@@ -38,6 +60,18 @@ func main() {
 		return
 	}
 
+	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
+	if logfile != "" {
+		file, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+		if err != nil {
+			log.Fatal("Failed to setup log file: ", err)
+		}
+		defer file.Close()
+		log.SetOutput(file)
+	} else {
+		log.SetOutput(os.Stdout)
+	}
+
 	configPath, err := os.UserConfigDir()
 	if err != nil {
 		log.Fatal("Failed to get config dir: ", err)

src/parser/feed.go

@@ -9,6 +9,8 @@ import (
 	"net/url"
 	"strings"
 	"time"
+
+	"github.com/nkanaev/yarr/src/content/htmlutil"
 )
 
 var UnknownFormat = errors.New("unknown feed format")
@@ -18,6 +20,11 @@ type processor func(r io.Reader) (*Feed, error)
 func sniff(lookup string) (string, processor) {
 	lookup = strings.TrimSpace(lookup)
 	lookup = strings.TrimLeft(lookup, "\x00\xEF\xBB\xBF\xFE\xFF")
+
+	if len(lookup) < 0 {
+		return "", nil
+	}
+
 	switch lookup[0] {
 	case '<':
 		decoder := xmlDecoder(strings.NewReader(lookup))
@@ -75,7 +82,7 @@ func (feed *Feed) cleanup() {
 	for i, item := range feed.Items {
 		feed.Items[i].GUID = strings.TrimSpace(item.GUID)
 		feed.Items[i].URL = strings.TrimSpace(item.URL)
-		feed.Items[i].Title = strings.TrimSpace(item.Title)
+		feed.Items[i].Title = strings.TrimSpace(htmlutil.ExtractText(item.Title))
 		feed.Items[i].Content = strings.TrimSpace(item.Content)
 
 		if item.ImageURL != "" && strings.Contains(item.Content, item.ImageURL) {

src/parser/rss.go

@@ -71,7 +71,7 @@ func ParseRSS(r io.Reader) (*Feed, error) {
 	for _, srcitem := range srcfeed.Items {
 		podcastURL := ""
 		for _, e := range srcitem.Enclosures {
-			if e.Type == "audio/mpeg" || e.Type == "audio/x-m4a" {
+			if strings.HasPrefix(e.Type, "audio/") {
 				podcastURL = e.URL
 
 				if srcitem.OrigEnclosureLink != "" && strings.Contains(podcastURL, path.Base(srcitem.OrigEnclosureLink)) {

src/parser/rss_test.go

@@ -136,6 +136,26 @@ func TestRSSPodcast(t *testing.T) {
 	}
 }
 
+func TestRSSOpusPodcast(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0">
+			<channel>
+				<item>
+					<enclosure length="100500" type="audio/opus" url="http://example.com/audio.ext"/>
+				</item>
+			</channel>
+		</rss>
+	`))
+	have := feed.Items[0].AudioURL
+	want := "http://example.com/audio.ext"
+	if want != have {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.FailNow()
+	}
+}
+
 // found in: https://podcast.cscript.site/podcast.xml
 func TestRSSPodcastDuplicated(t *testing.T) {
 	feed, _ := Parse(strings.NewReader(`
@@ -160,3 +180,26 @@ func TestRSSPodcastDuplicated(t *testing.T) {
 		t.Fatal("item.audio_url must be unset if present in the content")
 	}
 }
+
+func TestRSSTitleHTMLTags(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+					<title>&lt;p&gt;title in p&lt;/p&gt;</title>
+				</item>
+				<item>
+					<title>very &lt;strong&gt;strong&lt;/strong&gt; title</title>
+				</item>
+			</channel>
+		</rss>
+	`))
+	have := []string{feed.Items[0].Title, feed.Items[1].Title}
+	want := []string{"title in p", "very strong title"}
+	for i := 0; i < len(want); i++ {
+		if want[i] != have[i] {
+			t.Errorf("title doesn't match\nwant: %#v\nhave: %#v\n", want[i], have[i])
+		}
+	}
+}

src/platform/fixconsole_default.go

@@ -0,0 +1,14 @@
+// +build !windows
+
+package platform
+
+// On non-windows platforms, we don't need to do anything. The console
+// starts off attached already, if it exists.
+
+func AttachConsole() error {
+	return nil
+}
+
+func FixConsoleIfNeeded() error {
+	return nil
+}

src/platform/fixconsole_windows.go

@@ -0,0 +1,131 @@
+package platform
+
+import (
+	"fmt"
+	"golang.org/x/sys/windows"
+	"os"
+	"syscall"
+)
+
+func AttachConsole() error {
+	const ATTACH_PARENT_PROCESS = ^uintptr(0)
+	proc := syscall.MustLoadDLL("kernel32.dll").MustFindProc("AttachConsole")
+	r1, _, err := proc.Call(ATTACH_PARENT_PROCESS)
+	if r1 == 0 {
+		errno, ok := err.(syscall.Errno)
+		if ok && errno == windows.ERROR_INVALID_HANDLE {
+			// console handle doesn't exist; not a real
+			// error, but the console handle will be
+			// invalid.
+			return nil
+		}
+		return err
+	} else {
+		return nil
+	}
+}
+
+var oldStdin, oldStdout, oldStderr *os.File
+
+// Windows console output is a mess.
+//
+// If you compile as "-H windows", then if you launch your program without
+// a console, Windows forcibly creates one to use as your stdin/stdout, which
+// is silly for a GUI app, so we can't do that.
+//
+// If you compile as "-H windowsgui", then it doesn't create a console for
+// your app... but also doesn't provide a working stdin/stdout/stderr even if
+// you *did* launch from the console.  However, you can use AttachConsole()
+// to get a handle to your parent process's console, if any, and then
+// os.NewFile() to turn that handle into a fd usable as stdout/stderr.
+//
+// However, then you have the problem that if you redirect stdout or stderr
+// from the shell, you end up ignoring the redirection by forcing it to the
+// console.
+//
+// To fix *that*, we have to detect whether there was a pre-existing stdout
+// or not. We can check GetStdHandle(), which returns 0 for "should be
+// console" and nonzero for "already pointing at a file."
+//
+// Be careful though!  As soon as you run AttachConsole(), it resets *all*
+// the GetStdHandle() handles to point them at the console instead, thus
+// throwing away the original file redirects.  So we have to GetStdHandle()
+// *before* AttachConsole().
+//
+// For some reason, powershell redirections provide a valid file handle, but
+// writing to that handle doesn't write to the file.  I haven't found a way
+// to work around that.  (Windows 10.0.17763.379)
+//
+// Net result is as follows.
+// Before:
+//    SHELL            NON-REDIRECTED     REDIRECTED
+//    explorer.exe     no console         n/a
+//    cmd.exe          broken             works
+//    powershell       broken             broken
+//    WSL bash         broken             works
+// After
+//    SHELL            NON-REDIRECTED     REDIRECTED
+//    explorer.exe     no console         n/a
+//    cmd.exe          works              works
+//    powershell       works              broken
+//    WSL bash         works              works
+//
+// We don't seem to make anything worse, at least.
+func FixConsoleIfNeeded() error {
+	// Retain the original console objects, to prevent Go from automatically
+	// closing their file descriptors when they get garbage collected.
+	// You never want to close file descriptors 0, 1, and 2.
+	oldStdin, oldStdout, oldStderr = os.Stdin, os.Stdout, os.Stderr
+
+	stdin, _ := syscall.GetStdHandle(syscall.STD_INPUT_HANDLE)
+	stdout, _ := syscall.GetStdHandle(syscall.STD_OUTPUT_HANDLE)
+	stderr, _ := syscall.GetStdHandle(syscall.STD_ERROR_HANDLE)
+
+	var invalid syscall.Handle
+	con := invalid
+
+	if stdin == invalid || stdout == invalid || stderr == invalid {
+		err := AttachConsole()
+		if err != nil {
+			return fmt.Errorf("attachconsole: %v", err)
+		}
+
+		if stdin == invalid {
+			stdin, _ = syscall.GetStdHandle(syscall.STD_INPUT_HANDLE)
+		}
+		if stdout == invalid {
+			stdout, _ = syscall.GetStdHandle(syscall.STD_OUTPUT_HANDLE)
+			con = stdout
+		}
+		if stderr == invalid {
+			stderr, _ = syscall.GetStdHandle(syscall.STD_ERROR_HANDLE)
+			con = stderr
+		}
+	}
+
+	if con != invalid {
+		// Make sure the console is configured to convert
+		// \n to \r\n, like Go programs expect.
+		h := windows.Handle(con)
+		var st uint32
+		err := windows.GetConsoleMode(h, &st)
+		if err != nil {
+			return fmt.Errorf("GetConsoleMode: %v", err)
+		}
+		err = windows.SetConsoleMode(h, st&^windows.DISABLE_NEWLINE_AUTO_RETURN)
+		if err != nil {
+			return fmt.Errorf("SetConsoleMode: %v", err)
+		}
+	}
+
+	if stdin != invalid {
+		os.Stdin = os.NewFile(uintptr(stdin), "stdin")
+	}
+	if stdout != invalid {
+		os.Stdout = os.NewFile(uintptr(stdout), "stdout")
+	}
+	if stderr != invalid {
+		os.Stderr = os.NewFile(uintptr(stderr), "stderr")
+	}
+	return nil
+}

src/server/gzip/middleware.go

@@ -28,15 +28,16 @@ func (rw *gzipResponseWriter) WriteHeader(statusCode int) {
 }
 
 func Middleware(c *router.Context) {
-	if strings.Contains(c.Req.Header.Get("Accept-Encoding"), "gzip") {
-		gz := &gzipResponseWriter{out: gzip.NewWriter(c.Out), src: c.Out}
-		defer gz.out.Close()
-
-		c.Out.Header().Set("Content-Encoding", "gzip")
-		c.Out = gz
+	if !strings.Contains(c.Req.Header.Get("Accept-Encoding"), "gzip") {
 		c.Next()
-
 		return
 	}
+
+	gz := &gzipResponseWriter{out: gzip.NewWriter(c.Out), src: c.Out}
+	defer gz.out.Close()
+
+	c.Out.Header().Set("Content-Encoding", "gzip")
+	c.Out = gz
+
 	c.Next()
 }

src/server/opml/read.go

@@ -13,6 +13,7 @@ type opml struct {
 type outline struct {
 	Type     string    `xml:"type,attr,omitempty"`
 	Title    string    `xml:"text,attr"`
+	Title2   string    `xml:"title,attr,omitempty"`
 	FeedUrl  string    `xml:"xmlUrl,attr,omitempty"`
 	SiteUrl  string    `xml:"htmlUrl,attr,omitempty"`
 	Outlines []outline `xml:"outline,omitempty"`
@@ -21,14 +22,18 @@ type outline struct {
 func buildFolder(title string, outlines []outline) Folder {
 	folder := Folder{Title: title}
 	for _, outline := range outlines {
-		if outline.Type == "rss" {
+		if outline.Type == "rss" || outline.FeedUrl != "" {
 			folder.Feeds = append(folder.Feeds, Feed{
 				Title:   outline.Title,
 				FeedUrl: outline.FeedUrl,
 				SiteUrl: outline.SiteUrl,
 			})
 		} else {
-			subfolder := buildFolder(outline.Title, outline.Outlines)
+			title := outline.Title
+			if title == "" {
+				title = outline.Title2
+			}
+			subfolder := buildFolder(title, outline.Outlines)
 			folder.Folders = append(folder.Folders, subfolder)
 		}
 	}

src/server/opml/read_test.go

@@ -56,3 +56,34 @@ func TestParse(t *testing.T) {
 		t.Fatal("invalid opml")
 	}
 }
+
+func TestParseFallback(t *testing.T) {
+	// as reported in https://github.com/nkanaev/yarr/pull/56
+	// the feed below comes without `outline[text]` & `outline[type=rss]` attributes
+	have, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<opml xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" version="1.0">
+		  <head>
+			<title>Newsflow</title>
+		  </head>
+		  <body>
+			<outline title="foldertitle">
+				<outline htmlUrl="https://example.com" text="feedtext" title="feedtitle" xmlUrl="https://example.com/feed.xml" />
+			</outline>
+		  </body>
+		</opml>
+	`))
+	want := Folder{
+		Folders: []Folder{{
+			Title: "foldertitle",
+			Feeds: []Feed{
+				{Title: "feedtext", FeedUrl: "https://example.com/feed.xml", SiteUrl: "https://example.com"},
+			},
+		}},
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid opml")
+	}
+}

src/server/router/context.go

@@ -29,15 +29,15 @@ func (c *Context) JSON(status int, data interface{}) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	c.Out.WriteHeader(status)
 	c.Out.Header().Set("Content-Type", "application/json; charset=utf-8")
+	c.Out.WriteHeader(status)
 	c.Out.Write(body)
 	c.Out.Write([]byte("\n"))
 }
 
 func (c *Context) HTML(status int, tmpl *template.Template, data interface{}) {
-	c.Out.WriteHeader(status)
 	c.Out.Header().Set("Content-Type", "text/html")
+	c.Out.WriteHeader(status)
 	tmpl.Execute(c.Out, data)
 }
 

src/server/routes.go

@@ -1,12 +1,14 @@
 package server
 
 import (
+	"crypto/md5"
 	"encoding/json"
+	"fmt"
 	"log"
-	"math"
 	"net/http"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 
 	"github.com/nkanaev/yarr/src/assets"
@@ -143,19 +145,51 @@ func (s *Server) handleFeedErrors(c *router.Context) {
 	c.JSON(http.StatusOK, errors)
 }
 
+type feedicon struct {
+	ctype string
+	bytes []byte
+	etag  string
+}
+
 func (s *Server) handleFeedIcon(c *router.Context) {
 	id, err := c.VarInt64("id")
 	if err != nil {
 		c.Out.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	feed := s.db.GetFeed(id)
-	if feed != nil && feed.Icon != nil {
-		c.Out.Header().Set("Content-Type", http.DetectContentType(*feed.Icon))
-		c.Out.Write(*feed.Icon)
-	} else {
-		c.Out.WriteHeader(http.StatusNotFound)
+
+	cachekey := "icon:" + strconv.FormatInt(id, 10)
+	cachedat := s.cache[cachekey]
+	if cachedat == nil {
+		feed := s.db.GetFeed(id)
+		if feed == nil || feed.Icon == nil {
+			c.Out.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		hash := md5.New()
+		hash.Write(*feed.Icon)
+
+		etag := fmt.Sprintf("%x", hash.Sum(nil))[:16]
+
+		cachedat = feedicon{
+			ctype: http.DetectContentType(*feed.Icon),
+			bytes: *(*feed).Icon,
+			etag:  etag,
+		}
+		s.cache[cachekey] = cachedat
 	}
+
+	icon := cachedat.(feedicon)
+
+	if c.Req.Header.Get("If-None-Match") == icon.etag {
+		c.Out.WriteHeader(http.StatusNotModified)
+		return
+	}
+
+	c.Out.Header().Set("Content-Type", icon.ctype)
+	c.Out.Header().Set("Etag", icon.etag)
+	c.Out.Write(icon.bytes)
 }
 
 func (s *Server) handleFeedList(c *router.Context) {
@@ -190,7 +224,7 @@ func (s *Server) handleFeedList(c *router.Context) {
 
 			c.JSON(http.StatusOK, map[string]interface{}{
 				"status": "success",
-				"feed": feed,
+				"feed":   feed,
 			})
 		default:
 			c.JSON(http.StatusOK, map[string]string{"status": "notfound"})
@@ -272,11 +306,8 @@ func (s *Server) handleItem(c *router.Context) {
 func (s *Server) handleItemList(c *router.Context) {
 	if c.Req.Method == "GET" {
 		perPage := 20
-		curPage := 1
 		query := c.Req.URL.Query()
-		if page, err := c.QueryInt64("page"); err == nil {
-			curPage = int(page)
-		}
+
 		filter := storage.ItemFilter{}
 		if folderID, err := c.QueryInt64("folder_id"); err == nil {
 			filter.FolderID = &folderID
@@ -284,6 +315,9 @@ func (s *Server) handleItemList(c *router.Context) {
 		if feedID, err := c.QueryInt64("feed_id"); err == nil {
 			filter.FeedID = &feedID
 		}
+		if after, err := c.QueryInt64("after"); err == nil {
+			filter.After = &after
+		}
 		if status := query.Get("status"); len(status) != 0 {
 			statusValue := storage.StatusValues[status]
 			filter.Status = &statusValue
@@ -292,14 +326,16 @@ func (s *Server) handleItemList(c *router.Context) {
 			filter.Search = &search
 		}
 		newestFirst := query.Get("oldest_first") != "true"
-		items := s.db.ListItems(filter, (curPage-1)*perPage, perPage, newestFirst)
-		count := s.db.CountItems(filter)
+
+		items := s.db.ListItems(filter, perPage+1, newestFirst)
+		hasMore := false
+		if len(items) == perPage+1 {
+			hasMore = true
+			items = items[:perPage]
+		}
 		c.JSON(http.StatusOK, map[string]interface{}{
-			"page": map[string]int{
-				"cur": curPage,
-				"num": int(math.Ceil(float64(count) / float64(perPage))),
-			},
 			"list": items,
+			"has_more": hasMore,
 		})
 	} else if c.Req.Method == "PUT" {
 		filter := storage.MarkFilter{}
@@ -416,7 +452,7 @@ func (s *Server) handlePageCrawl(c *router.Context) {
 
 	if content := silo.VideoIFrame(url); content != "" {
 		c.JSON(http.StatusOK, map[string]string{
-			"content": content,
+			"content": sanitizer.Sanitize(url, content),
 		})
 		return
 	}

src/server/routes_test.go

@@ -1,8 +1,16 @@
 package server
 
 import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
 	"net/http/httptest"
+	"os"
+	"reflect"
 	"testing"
+
+	"github.com/nkanaev/yarr/src/storage"
 )
 
 func TestStatic(t *testing.T) {
@@ -43,3 +51,64 @@ func TestStaticBanTemplates(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+func TestIndexGzipped(t *testing.T) {
+	log.SetOutput(io.Discard)
+	db, _ := storage.New(":memory:")
+	log.SetOutput(os.Stderr)
+	handler := NewServer(db, "127.0.0.1:8000").handler()
+	url := "/"
+
+	recorder := httptest.NewRecorder()
+	request := httptest.NewRequest("GET", url, nil)
+	request.Header.Set("accept-encoding", "gzip")
+	handler.ServeHTTP(recorder, request)
+	response := recorder.Result()
+	if response.StatusCode != 200 {
+		t.FailNow()
+	}
+	if response.Header.Get("content-encoding") != "gzip" {
+		t.Errorf("invalid content-encoding header: %#v", response.Header.Get("content-encoding"))
+	}
+	if response.Header.Get("content-type") != "text/html" {
+		t.Errorf("invalid content-type header: %#v", response.Header.Get("content-type"))
+	}
+}
+
+func TestFeedIcons(t *testing.T) {
+	log.SetOutput(io.Discard)
+	db, _ := storage.New(":memory:")
+	icon := []byte("test")
+	feed := db.CreateFeed("", "", "", "", nil)
+	db.UpdateFeedIcon(feed.Id, &icon)
+	log.SetOutput(os.Stderr)
+
+	recorder := httptest.NewRecorder()
+	url := fmt.Sprintf("/api/feeds/%d/icon", feed.Id)
+	request := httptest.NewRequest("GET", url, nil)
+
+	handler := NewServer(db, "127.0.0.1:8000").handler()
+	handler.ServeHTTP(recorder, request)
+	response := recorder.Result()
+
+	if response.StatusCode != http.StatusOK {
+		t.Fatal()
+	}
+	body, _ := io.ReadAll(response.Body)
+	if !reflect.DeepEqual(body, icon) {
+		t.Fatal()
+	}
+	if response.Header.Get("Etag") == "" {
+		t.Fatal()
+	}
+
+	recorder2 := httptest.NewRecorder()
+	request2 := httptest.NewRequest("GET", url, nil)
+	request2.Header.Set("If-None-Match", response.Header.Get("Etag"))
+	handler.ServeHTTP(recorder2, request2)
+	response2 := recorder2.Result()
+
+	if response2.StatusCode != http.StatusNotModified {
+		t.Fatal("got", response2.StatusCode)
+	}
+}

src/server/server.go

@@ -12,6 +12,7 @@ type Server struct {
 	Addr   string
 	db     *storage.Storage
 	worker *worker.Worker
+	cache  map[string]interface{}
 
 	BasePath string
 
@@ -28,6 +29,7 @@ func NewServer(db *storage.Storage, addr string) *Server {
 		db:     db,
 		Addr:   addr,
 		worker: worker.NewWorker(db),
+		cache:  make(map[string]interface{}),
 	}
 }
 

src/storage/feed.go

@@ -76,10 +76,10 @@ func (s *Storage) UpdateFeedIcon(feedId int64, icon *[]byte) bool {
 }
 
 func (s *Storage) ListFeeds() []Feed {
-	result := make([]Feed, 0, 0)
+	result := make([]Feed, 0)
 	rows, err := s.db.Query(`
 		select id, folder_id, title, description, link, feed_link,
-		       ifnull(icon, '') != '' as has_icon
+		       ifnull(length(icon), 0) > 0 as has_icon
 		from feeds
 		order by title collate nocase
 	`)
@@ -107,6 +107,36 @@ func (s *Storage) ListFeeds() []Feed {
 	return result
 }
 
+func (s *Storage) ListFeedsMissingIcons() []Feed {
+	result := make([]Feed, 0)
+	rows, err := s.db.Query(`
+		select id, folder_id, title, description, link, feed_link
+		from feeds
+		where icon is null
+	`)
+	if err != nil {
+		log.Print(err)
+		return result
+	}
+	for rows.Next() {
+		var f Feed
+		err = rows.Scan(
+			&f.Id,
+			&f.FolderId,
+			&f.Title,
+			&f.Description,
+			&f.Link,
+			&f.FeedLink,
+		)
+		if err != nil {
+			log.Print(err)
+			return result
+		}
+		result = append(result, f)
+	}
+	return result
+}
+
 func (s *Storage) GetFeed(id int64) *Feed {
 	var f Feed
 	err := s.db.QueryRow(`

src/storage/item.go

@@ -61,6 +61,7 @@ type ItemFilter struct {
 	FeedID   *int64
 	Status   *ItemStatus
 	Search   *string
+	After  *int64
 }
 
 type MarkFilter struct {
@@ -106,7 +107,7 @@ func (s *Storage) CreateItems(items []Item) bool {
 	return true
 }
 
-func listQueryPredicate(filter ItemFilter) (string, []interface{}) {
+func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interface{}) {
 	cond := make([]string, 0)
 	args := make([]interface{}, 0)
 	if filter.FolderID != nil {
@@ -131,6 +132,14 @@ func listQueryPredicate(filter ItemFilter) (string, []interface{}) {
 		cond = append(cond, "i.search_rowid in (select rowid from search where search match ?)")
 		args = append(args, strings.Join(terms, " "))
 	}
+	if filter.After != nil {
+		compare := ">"
+		if newestFirst {
+			compare = "<"
+		}
+		cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = ?)", compare))
+		args = append(args, *filter.After)
+	}
 
 	predicate := "1"
 	if len(cond) > 0 {
@@ -140,13 +149,13 @@ func listQueryPredicate(filter ItemFilter) (string, []interface{}) {
 	return predicate, args
 }
 
-func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bool) []Item {
-	predicate, args := listQueryPredicate(filter)
+func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []Item {
+	predicate, args := listQueryPredicate(filter, newestFirst)
 	result := make([]Item, 0, 0)
 
-	order := "date desc"
+	order := "date desc, id desc"
 	if !newestFirst {
-		order = "date asc"
+		order = "date asc, id asc"
 	}
 
 	query := fmt.Sprintf(`
@@ -157,8 +166,8 @@ func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bo
 		from items i
 		where %s
 		order by %s
-		limit %d offset %d
-		`, predicate, order, limit, offset)
+		limit %d
+		`, predicate, order, limit)
 	rows, err := s.db.Query(query, args...)
 	if err != nil {
 		log.Print(err)
@@ -199,28 +208,13 @@ func (s *Storage) GetItem(id int64) *Item {
 	return i
 }
 
-func (s *Storage) CountItems(filter ItemFilter) int64 {
-	predicate, args := listQueryPredicate(filter)
-	query := fmt.Sprintf(`
-		select count(i.id)
-		from items i
-		where %s`, predicate)
-	row := s.db.QueryRow(query, args...)
-	if row != nil {
-		var result int64
-		row.Scan(&result)
-		return result
-	}
-	return 0
-}
-
 func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
 	_, err := s.db.Exec(`update items set status = ? where id = ?`, status, item_id)
 	return err == nil
 }
 
 func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
-	predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID})
+	predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID}, false)
 	query := fmt.Sprintf(`
 		update items as i set status = %d
 		where %s and i.status != %d

src/storage/item_test.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"log"
 	"reflect"
 	"testing"
 	"time"
@@ -44,18 +45,18 @@ func testItemsSetup(db *Storage) testItemScope {
 	db.CreateItems([]Item{
 		// feed11
 		{GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)},
-		{GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)},
-		{GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)},
+		{GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)},  // read
+		{GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)},  // starred
 		// feed12
 		{GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)},
-		{GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)},
+		{GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)},  // read
 		// feed21
-		{GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)},
-		{GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)},
+		{GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)},  // read
+		{GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)},  // starred
 		// feed01
 		{GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)},
-		{GUID: "item012", FeedId: feed01.Id, Title: "title012", Date: now.Add(time.Hour * 24 * 9)},
-		{GUID: "item013", FeedId: feed01.Id, Title: "title013", Date: now.Add(time.Hour * 24 * 10)},
+		{GUID: "item012", FeedId: feed01.Id, Title: "title012", Date: now.Add(time.Hour * 24 * 9)},  // read
+		{GUID: "item013", FeedId: feed01.Id, Title: "title013", Date: now.Add(time.Hour * 24 * 10)}, // starred
 	})
 	db.db.Exec(`update items set status = ? where guid in ("item112", "item122", "item211", "item012")`, READ)
 	db.db.Exec(`update items set status = ? where guid in ("item113", "item212", "item013")`, STARRED)
@@ -70,7 +71,25 @@ func testItemsSetup(db *Storage) testItemScope {
 	}
 }
 
-func testItemGuids(items []Item) []string {
+func getItem(db *Storage, guid string) *Item {
+	i := &Item{}
+	err := db.db.QueryRow(`
+		select
+			i.id, i.guid, i.feed_id, i.title, i.link, i.content,
+			i.date, i.status, i.image, i.podcast_url
+		from items i
+		where i.guid = ?
+	`, guid).Scan(
+		&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
+		&i.Date, &i.Status, &i.ImageURL, &i.AudioURL,
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return i
+}
+
+func getItemGuids(items []Item) []string {
 	guids := make([]string, 0)
 	for _, item := range items {
 		guids = append(guids, item.GUID)
@@ -84,7 +103,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by folder_id
 
-	have := testItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 0, 10, false))
+	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false))
 	want := []string{"item111", "item112", "item113", "item121", "item122"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -92,7 +111,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = testItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false))
 	want = []string{"item211", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -102,7 +121,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by feed_id
 
-	have = testItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false))
 	want = []string{"item111", "item112", "item113"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -110,7 +129,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = testItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false))
 	want = []string{"item011", "item012", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -121,7 +140,7 @@ func TestListItems(t *testing.T) {
 	// filter by status
 
 	var starred ItemStatus = STARRED
-	have = testItemGuids(db.ListItems(ItemFilter{Status: &starred}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false))
 	want = []string{"item113", "item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -130,7 +149,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	var unread ItemStatus = UNREAD
-	have = testItemGuids(db.ListItems(ItemFilter{Status: &unread}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false))
 	want = []string{"item111", "item121", "item011"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -138,9 +157,9 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	// filter by offset,limit
+	// limit
 
-	have = testItemGuids(db.ListItems(ItemFilter{}, 0, 2, false))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false))
 	want = []string{"item111", "item112"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -148,18 +167,10 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = testItemGuids(db.ListItems(ItemFilter{}, 2, 3, false))
-	want = []string{"item113", "item121", "item122"}
-	if !reflect.DeepEqual(have, want) {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
 	// filter by search
 	db.SyncSearch()
 	search1 := "title111"
-	have = testItemGuids(db.ListItems(ItemFilter{Search: &search1}, 0, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true))
 	want = []string{"item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -168,7 +179,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	// sort by date
-	have = testItemGuids(db.ListItems(ItemFilter{}, 0, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true))
 	want = []string{"item013", "item012", "item011", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -177,72 +188,37 @@ func TestListItems(t *testing.T) {
 	}
 }
 
-func TestCountItems(t *testing.T) {
+func TestListItemsPaginated(t *testing.T) {
 	db := testDB()
-	scope := testItemsSetup(db)
+	testItemsSetup(db)
 
-	have := db.CountItems(ItemFilter{})
-	want := int64(10)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-	
-	// folders
+	item012 := getItem(db, "item012")
+	item121 := getItem(db, "item121")
 
-	have = db.CountItems(ItemFilter{FolderID: &scope.folder1.Id})
-	want = int64(5)
-	if have != want {
+	// all, newest first
+	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true))
+	want := []string{"item011", "item212", "item211"}
+	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.Fail()
 	}
 
-	have = db.CountItems(ItemFilter{FolderID: &scope.folder2.Id})
-	want = int64(2)
-	if have != want {
+	// unread, newest first
+	unread := UNREAD
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true))
+	want = []string{"item011", "item121", "item111"}
+	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.Fail()
 	}
 
-	// feeds
-
-	have = db.CountItems(ItemFilter{FeedID: &scope.feed21.Id})
-	want = int64(2)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
-	have = db.CountItems(ItemFilter{FeedID: &scope.feed01.Id})
-	want = int64(3)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
-	// statuses
-
-	var unread ItemStatus = UNREAD
-	have = db.CountItems(ItemFilter{Status: &unread})
-	want = int64(3)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
-	// search
-
-	db.SyncSearch()
-	search := "title0"
-	have = db.CountItems(ItemFilter{Search: &search})
-	want = int64(3)
-	if have != want {
+	// starred, oldest first
+	starred := STARRED
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false))
+	want = []string{"item212", "item013"}
+	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.Fail()
@@ -256,7 +232,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db1 := testDB()
 	testItemsSetup(db1)
 	db1.MarkItemsRead(MarkFilter{})
-	have := testItemGuids(db1.ListItems(ItemFilter{Status: &read}, 0, 10, false))
+	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false))
 	want := []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item011", "item012",
@@ -270,7 +246,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db2 := testDB()
 	scope2 := testItemsSetup(db2)
 	db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
-	have = testItemGuids(db2.ListItems(ItemFilter{Status: &read}, 0, 10, false))
+	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false))
 	want = []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item012",
@@ -284,7 +260,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db3 := testDB()
 	scope3 := testItemsSetup(db3)
 	db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
-	have = testItemGuids(db3.ListItems(ItemFilter{Status: &read}, 0, 10, false))
+	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false))
 	want = []string{
 		"item111", "item112", "item122",
 		"item211", "item012",

src/storage/storage.go

@@ -15,6 +15,7 @@ func New(path string) (*Storage, error) {
 		return nil, err
 	}
 
+	// TODO: https://foxcpp.dev/articles/the-right-way-to-use-go-sqlite3
 	db.SetMaxOpenConns(1)
 
 	if err = migrate(db); err != nil {

src/worker/crawler.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/nkanaev/yarr/src/content/scraper"
@@ -38,7 +40,7 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
 		return nil, fmt.Errorf("status code %d", res.StatusCode)
 	}
 
-	body, err := charset.NewReader(res.Body, res.Header.Get("Content-Type"))
+	body, err := httpBody(res)
 	if err != nil {
 		return nil, err
 	}
@@ -76,8 +78,16 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
 	return result, nil
 }
 
-func findFavicon(websiteUrl, feedUrl string) (*[]byte, error) {
-	candidateUrls := make([]string, 0)
+var emptyIcon = make([]byte, 0)
+var imageTypes = map[string]bool{
+	"image/x-icon": true,
+	"image/png":    true,
+	"image/jpeg":   true,
+	"image/gif":    true,
+}
+
+func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
+	urls := make([]string, 0)
 
 	favicon := func(link string) string {
 		u, err := url.Parse(link)
@@ -87,49 +97,43 @@ func findFavicon(websiteUrl, feedUrl string) (*[]byte, error) {
 		return fmt.Sprintf("%s://%s/favicon.ico", u.Scheme, u.Host)
 	}
 
-	if len(websiteUrl) != 0 {
-		res, err := client.get(websiteUrl)
-		if err != nil {
-			return nil, err
-		}
-		body, err := ioutil.ReadAll(res.Body)
-		defer res.Body.Close()
-		if err != nil {
-			return nil, err
-		}
-		candidateUrls = append(candidateUrls, scraper.FindIcons(string(body), websiteUrl)...)
-		if c := favicon(websiteUrl); len(c) != 0 {
-			candidateUrls = append(candidateUrls, c)
-		}
-	}
-	if c := favicon(feedUrl); len(c) != 0 {
-		candidateUrls = append(candidateUrls, c)
-	}
-
-	imageTypes := [4]string{
-		"image/x-icon",
-		"image/png",
-		"image/jpeg",
-		"image/gif",
-	}
-	for _, url := range candidateUrls {
-		res, err := client.get(url)
-		if err != nil {
-			continue
-		}
-		defer res.Body.Close()
-		if res.StatusCode == 200 {
-			if content, err := ioutil.ReadAll(res.Body); err == nil {
-				ctype := http.DetectContentType(content)
-				for _, itype := range imageTypes {
-					if ctype == itype {
-						return &content, nil
-					}
+	if siteUrl != "" {
+		if res, err := client.get(siteUrl); err == nil {
+			defer res.Body.Close()
+			if body, err := ioutil.ReadAll(res.Body); err == nil {
+				urls = append(urls, scraper.FindIcons(string(body), siteUrl)...)
+				if c := favicon(siteUrl); c != "" {
+					urls = append(urls, c)
 				}
 			}
 		}
 	}
-	return nil, nil
+
+	if c := favicon(feedUrl); c != "" {
+		urls = append(urls, c)
+	}
+
+	for _, u := range urls {
+		res, err := client.get(u)
+		if err != nil {
+			continue
+		}
+		defer res.Body.Close()
+		if res.StatusCode != 200 {
+			continue
+		}
+
+		content, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			continue
+		}
+
+		ctype := http.DetectContentType(content)
+		if imageTypes[ctype] {
+			return &content, nil
+		}
+	}
+	return &emptyIcon, nil
 }
 
 func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
@@ -183,7 +187,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 		return nil, nil
 	}
 
-	body, err := charset.NewReader(res.Body, res.Header.Get("Content-Type"))
+	body, err := httpBody(res)
 	if err != nil {
 		return nil, err
 	}
@@ -202,3 +206,11 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 	feed.SetMissingDatesTo(time.Now())
 	return ConvertItems(feed.Items, f), nil
 }
+
+func httpBody(res *http.Response) (io.Reader, error) {
+	ctype := res.Header.Get("Content-Type")
+	if strings.Contains(ctype, "charset") {
+		return charset.NewReader(res.Body, ctype)
+	}
+	return res.Body, nil
+}

src/worker/worker.go

@@ -9,6 +9,8 @@ import (
 	"github.com/nkanaev/yarr/src/storage"
 )
 
+const NUM_WORKERS = 4
+
 type Worker struct {
 	db      *storage.Storage
 	pending *int32
@@ -39,10 +41,8 @@ func (w *Worker) StartFeedCleaner() {
 
 func (w *Worker) FindFavicons() {
 	go func() {
-		for _, feed := range w.db.ListFeeds() {
-			if !feed.HasIcon {
-				w.FindFeedFavicon(feed)
-			}
+		for _, feed := range w.db.ListFeedsMissingIcons() {
+			w.FindFeedFavicon(feed)
 		}
 	}()
 }
@@ -88,30 +88,34 @@ func (w *Worker) SetRefreshRate(minute int64) {
 }
 
 func (w *Worker) RefreshFeeds() {
-	log.Print("Refreshing feeds")
-	go w.refresher()
-}
-
-func (w *Worker) refresher() {
 	w.reflock.Lock()
+	defer w.reflock.Unlock()
 
-	w.db.ResetFeedErrors()
-
-	feeds := w.db.ListFeeds()
-	if len(feeds) == 0 {
+	if *w.pending > 0 {
+		log.Print("Refreshing already in progress")
 		return
 	}
 
+	feeds := w.db.ListFeeds()
+	if len(feeds) == 0 {
+		log.Print("Nothing to refresh")
+		return
+	}
+
+	log.Print("Refreshing feeds")
 	atomic.StoreInt32(w.pending, int32(len(feeds)))
+	go w.refresher(feeds)
+}
+
+func (w *Worker) refresher(feeds []storage.Feed) {
+	w.db.ResetFeedErrors()
 
 	srcqueue := make(chan storage.Feed, len(feeds))
 	dstqueue := make(chan []storage.Item)
 
-	// hardcoded to 4 workers ;)
-	go w.worker(srcqueue, dstqueue)
-	go w.worker(srcqueue, dstqueue)
-	go w.worker(srcqueue, dstqueue)
-	go w.worker(srcqueue, dstqueue)
+	for i := 0; i < NUM_WORKERS; i++ {
+		go w.worker(srcqueue, dstqueue)
+	}
 
 	for _, feed := range feeds {
 		srcqueue <- feed
@@ -119,14 +123,12 @@ func (w *Worker) refresher() {
 	for i := 0; i < len(feeds); i++ {
 		w.db.CreateItems(<-dstqueue)
 		atomic.AddInt32(w.pending, -1)
+		w.db.SyncSearch()
 	}
 	close(srcqueue)
 	close(dstqueue)
 
-	w.db.SyncSearch()
 	log.Printf("Finished refreshing %d feeds", len(feeds))
-
-	w.reflock.Unlock()
 }
 
 func (w *Worker) worker(srcqueue <-chan storage.Feed, dstqueue chan<- []storage.Item) {