reorganise build workflow

Bumps [golang.org/x/net](https://github.com/golang/net) from 0.33.0 to 0.36.0.
- [Commits](https://github.com/golang/net/compare/v0.33.0...v0.36.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/actions/prepare/action.yml

@@ -0,0 +1,25 @@
+name: Build & Upload
+inputs:
+  id:
+    description: artifact name
+    required: true
+  cmd:
+    description: command to run
+    required: true
+  out:
+    description: path to output file
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: compile
+      run: ${{ inputs.cmd }}
+      shell: bash
+    - name: archive
+      run: tar -cvf ${{ inputs.out }}.tar ${{ inputs.out }}
+      shell: bash
+    - name: upload
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.id }}
+        path: ${{ inputs.out }}.tar

.github/workflows/build-docker-image

@@ -0,0 +1,38 @@
+name: Publish Docker Image
+on:
+  push:
+    tags: [ 'v*.*.*', 'v*.*', 'v*', 'latest' ]
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: nkanaev/yarr
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          file: ./etc/dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/build.yml

@@ -1,144 +1,136 @@
-name: build
+name: Build
 
 on:
   push:
-    tags: ['v*', 'test*']
+    tags:
+      - v*
+  workflow_dispatch:
 
 jobs:
   build_macos:
     name: Build for MacOS
-    runs-on: macos-10.15
+    runs-on: macos-13
     steps:
-    - name: "Checkout"
-      uses: actions/checkout@v2
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Go
+      uses: actions/setup-go@v4
       with:
-        submodules: 'recursive'
-    - name: "Setup Go"
-      uses: actions/setup-go@v2
+        go-version: '^1.18'
+    - name: Build arm64 gui
+      uses: ./.github/actions/prepare
       with:
-        go-version: '^1.16'
-    - name: Cache Go Modules
-      uses: actions/cache@v2
+        id: darwin_arm64_gui
+        cmd: make darwin_arm64_gui
+        out: out/darwin_arm64_gui/yarr.app
+    - name: Build amd64 gui
+      uses: ./.github/actions/prepare
       with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-    - name: "Build"
-      run: make build_macos
-    - name: Upload
-      uses: actions/upload-artifact@v2
+        id: darwin_amd64_gui
+        cmd: make darwin_amd64_gui
+        out: out/darwin_amd64_gui/yarr.app
+    - name: Build arm64 cli
+      uses: ./.github/actions/prepare
       with:
-        name: macos
-        path: _output/macos/yarr.app
+        id: darwin_arm64
+        cmd: make darwin_arm64
+        out: out/darwin_arm64/yarr
+    - name: Build amd64 cli
+      uses: ./.github/actions/prepare
+      with:
+        id: darwin_amd64
+        cmd: make darwin_amd64
+        out: out/darwin_amd64/yarr
 
   build_windows:
     name: Build for Windows
-    runs-on: windows-2019
+    runs-on: windows-2022
     steps:
-    - name: "Checkout"
-      uses: actions/checkout@v2
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Go
+      uses: actions/setup-go@v4
       with:
-        submodules: 'recursive'
-    - name: "Setup Go"
-      uses: actions/setup-go@v2
+        go-version: '^1.18'
+    - name: Build amd64 gui
+      uses: ./.github/actions/prepare
       with:
-        go-version: '^1.16'
-    - name: Cache Go Modules
-      uses: actions/cache@v2
+        id: windows_amd64_gui
+        cmd: make windows_amd64_gui
+        out: out/windows_amd64_gui/yarr.exe
+    - name: Build arm64 gui
+      if: false
+      uses: ./.github/actions/prepare
       with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-    - name: "Build"
-      run: make build_windows
-    - name: Upload
-      uses: actions/upload-artifact@v2
-      with:
-        name: windows
-        path: _output/windows/yarr.exe
+        id: windows_arm64_gui
+        cmd: make windows_arm64_gui
+        out: out/windows_arm64_gui/yarr.exe
 
-  build_linux:
-    name: Build for Linux
-    runs-on: ubuntu-18.04
+  build_multi_cli:
+    name: Build for Windows/MacOS/Linux CLI
+    runs-on: ubuntu-22.04
     steps:
-    - name: "Checkout"
-      uses: actions/checkout@v2
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Setup Go
+      uses: actions/setup-go@v4
       with:
-        submodules: 'recursive'
-    - name: "Setup Go"
-      uses: actions/setup-go@v2
+        go-version: '^1.18'
+    - name: Setup Zig
+      uses: mlugg/setup-zig@v1
       with:
-        go-version: '^1.16'
-    - name: Cache Go Modules
-      uses: actions/cache@v2
+        version: 0.14.0
+    - name: Build linux/amd64
+      uses: ./.github/actions/prepare
       with:
-        path: ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-    - name: "Build"
-      run: make build_linux
-    - name: Upload
-      uses: actions/upload-artifact@v2
+        id: linux_amd64
+        cmd: make linux_amd64
+        out: out/linux_amd64/yarr
+    - name: Build linux/arm64
+      uses: ./.github/actions/prepare
       with:
-        name: linux
-        path: _output/linux/yarr
+        id: linux_arm64
+        cmd: make linux_arm64
+        out: out/linux_arm64/yarr
+    - name: Build linux/armv7
+      uses: ./.github/actions/prepare
+      with:
+        id: linux_armv7
+        cmd: make linux_armv7
+        out: out/linux_armv7/yarr
+    - name: Build windows/amd64
+      uses: ./.github/actions/prepare
+      with:
+        id: windows_amd64
+        cmd: make windows_amd64
+        out: out/windows_amd64/yarr
+    - name: Build windows/arm64
+      uses: ./.github/actions/prepare
+      with:
+        id: windows_arm64
+        cmd: make windows_arm64
+        out: out/windows_arm64/yarr
 
   create_release:
     name: Create Release
     runs-on: ubuntu-latest
-    if: ${{ !contains(github.ref, 'test') }}
-    needs: [build_macos, build_windows, build_linux]
+    if: ${{ startsWith(github.ref, 'refs/tags/') }}
+    needs: [build_macos, build_windows, build_multi_cli]
     steps:
-    - name: Create Release
-      uses: actions/create-release@v1
-      id: create_release
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-          tag_name: ${{ github.ref }}
-          release_name: ${{ github.ref }}
-          draft: true
-          prerelease: true
     - name: Download Artifacts
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4.1.7
       with:
         path: .
     - name: Preparation
       run: |
         ls -R
-        chmod u+x macos/Contents/MacOS/yarr
-        chmod u+x linux/yarr
-
-        mv macos yarr.app     && zip -r yarr-macos.zip yarr.app
-        mv windows/yarr.exe . && zip yarr-windows.zip yarr.exe
-        mv linux/yarr .       && zip yarr-linux.zip yarr
-    - name: Upload MacOS
-      uses: actions/upload-release-asset@v1
+    - name: Upload Release
+      if: false
+      uses: softprops/action-gh-release@v2
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./yarr-macos.zip
-          asset_name: yarr-${{ github.ref }}-macos64.zip
-          asset_content_type: application/zip
-    - name: Upload Windows
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./yarr-windows.zip
-          asset_name: yarr-${{ github.ref }}-windows32.zip
-          asset_content_type: application/zip
-    - name: Upload Linux
-      uses: actions/upload-release-asset@v1
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./yarr-linux.zip
-          asset_name: yarr-${{ github.ref }}-linux32.zip
-          asset_content_type: application/zip
+          draft: true
+          prerelease: true
+          files: |
+            *.zip

.github/workflows/test.yml

@@ -0,0 +1,19 @@
+name: Test
+
+on: push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '^1.18'
+
+    - name: Run tests
+      run: make test

.gitignore

@@ -1,7 +1,8 @@
-/server/assets.go
-/gofeed
 /_output
+/out
 /yarr
 *.db
+*.db-shm
+*.db-wal
 *.syso
 versioninfo.rc

bin/generate_versioninfo.go

@@ -1,48 +0,0 @@
-package main
-
-import (
-	"flag"
-	"io/ioutil"
-	"strings"
-)
-
-var rsrc = `1 VERSIONINFO
-FILEVERSION     {VERSION_COMMA},0,0
-PRODUCTVERSION  {VERSION_COMMA},0,0
-BEGIN
-  BLOCK "StringFileInfo"
-  BEGIN
-    BLOCK "080904E4"
-    BEGIN
-      VALUE "CompanyName", "Old MacDonald's Farm"
-      VALUE "FileDescription", "Yet another RSS reader"
-      VALUE "FileVersion", "{VERSION}"
-      VALUE "InternalName", "yarr"
-      VALUE "LegalCopyright", "nkanaev"
-      VALUE "OriginalFilename", "yarr.exe"
-      VALUE "ProductName", "yarr"
-      VALUE "ProductVersion", "{VERSION}"
-    END
-  END
-  BLOCK "VarFileInfo"
-  BEGIN
-    VALUE "Translation", 0x809, 1252
-  END
-END
-
-1 ICON "icon.ico"
-`
-
-func main() {
-	var version, outfile string
-	flag.StringVar(&version, "version", "0.0", "")
-	flag.StringVar(&outfile, "outfile", "versioninfo.rc", "")
-	flag.Parse()
-
-	version_comma := strings.ReplaceAll(version, ".", ",")
-
-	out := strings.ReplaceAll(rsrc, "{VERSION}", version)
-	out = strings.ReplaceAll(out, "{VERSION_COMMA}", version_comma)
-
-	ioutil.WriteFile(outfile, []byte(out), 0644)
-}

bin/package_macos.go

@@ -1,99 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"os/exec"
-	"path"
-	"strconv"
-	"strings"
-)
-
-var plist = `<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>CFBundleName</key>
-	<string>yarr</string>
-	<key>CFBundleDisplayName</key>
-	<string>yarr</string>
-	<key>CFBundleIdentifier</key>
-	<string>nkanaev.yarr</string>
-	<key>CFBundleVersion</key>
-	<string>VERSION</string>
-	<key>CFBundlePackageType</key>
-	<string>APPL</string>
-	<key>CFBundleExecutable</key>
-	<string>yarr</string>
-
-	<key>CFBundleIconFile</key>
-	<string>icon</string>
-	<key>LSApplicationCategoryType</key>
-	<string>public.app-category.news</string>
-
-	<key>NSHighResolutionCapable</key>
-	<string>True</string>
-
-	<key>LSMinimumSystemVersion</key>
-	<string>10.13</string>
-	<key>LSUIElement</key>
-	<true/>
-	<key>NSHumanReadableCopyright</key>
-	<string>Copyright © 2020 nkanaev. All rights reserved.</string>
-</dict>
-</plist>
-`
-
-func run(cmd ...string) {
-	fmt.Println(cmd)
-	err := exec.Command(cmd[0], cmd[1:]...).Run()
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func main() {
-	var version, outdir string
-	flag.StringVar(&version, "version", "0.0", "")
-	flag.StringVar(&outdir, "outdir", "", "")
-	flag.Parse()
-
-	outfile := "yarr"
-
-	binDir := path.Join(outdir, "yarr.app", "Contents/MacOS")
-	resDir := path.Join(outdir, "yarr.app", "Contents/Resources")
-
-	plistFile := path.Join(outdir, "yarr.app", "Contents/Info.plist")
-	pkginfoFile := path.Join(outdir, "yarr.app", "Contents/PkgInfo")
-
-	os.MkdirAll(binDir, 0700)
-	os.MkdirAll(resDir, 0700)
-
-	f, _ := ioutil.ReadFile(path.Join(outdir, outfile))
-	ioutil.WriteFile(path.Join(binDir, outfile), f, 0755)
-
-	ioutil.WriteFile(plistFile, []byte(strings.Replace(plist, "VERSION", version, 1)), 0644)
-	ioutil.WriteFile(pkginfoFile, []byte("APPL????"), 0644)
-
-	iconFile := path.Join(outdir, "icon.png")
-	iconsetDir := path.Join(outdir, "icon.iconset")
-	os.Mkdir(iconsetDir, 0700)
-
-	for _, res := range []int{1024, 512, 256, 128, 64, 32, 16} {
-		outfile := fmt.Sprintf("icon_%dx%d.png", res, res)
-		if res == 1024 || res == 64 {
-			outfile = fmt.Sprintf("icon_%dx%d@2x.png", res/2, res/2)
-		}
-		cmd := []string{
-			"sips", "-s", "format", "png", "--resampleWidth", strconv.Itoa(res),
-			iconFile, "--out", path.Join(iconsetDir, outfile),
-		}
-		run(cmd...)
-	}
-
-	icnsFile := path.Join(resDir, "icon.icns")
-	run("iconutil", "-c", "icns", iconsetDir, "-o", icnsFile)
-}

cmd/yarr/main.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/nkanaev/yarr/src/platform"
+	"github.com/nkanaev/yarr/src/server"
+	"github.com/nkanaev/yarr/src/storage"
+	"github.com/nkanaev/yarr/src/worker"
+)
+
+var Version string = "0.0"
+var GitHash string = "unknown"
+
+var OptList = make([]string, 0)
+
+func opt(envVar, defaultValue string) string {
+	OptList = append(OptList, envVar)
+	value := os.Getenv(envVar)
+	if value != "" {
+		return value
+	}
+	return defaultValue
+}
+
+func parseAuthfile(authfile io.Reader) (username, password string, err error) {
+	scanner := bufio.NewScanner(authfile)
+	for scanner.Scan() {
+		line := scanner.Text()
+		parts := strings.SplitN(line, ":", 2)
+		if len(parts) != 2 {
+			return "", "", fmt.Errorf("wrong syntax (expected `username:password`)")
+		}
+		username = parts[0]
+		password = parts[1]
+		break
+	}
+	return username, password, nil
+}
+
+func main() {
+	platform.FixConsoleIfNeeded()
+
+	var addr, db, authfile, auth, certfile, keyfile, basepath, logfile string
+	var ver, open bool
+
+	flag.CommandLine.SetOutput(os.Stdout)
+
+	flag.Usage = func() {
+		out := flag.CommandLine.Output()
+		fmt.Fprintf(out, "Usage of %s:\n", os.Args[0])
+		flag.PrintDefaults()
+		fmt.Fprintln(out, "\nThe environmental variables, if present, will be used to provide\nthe default values for the params above:")
+		fmt.Fprintln(out, " ", strings.Join(OptList, ", "))
+	}
+
+	flag.StringVar(&addr, "addr", opt("YARR_ADDR", "127.0.0.1:7070"), "address to run server on")
+	flag.StringVar(&basepath, "base", opt("YARR_BASE", ""), "base path of the service url")
+	flag.StringVar(&authfile, "auth-file", opt("YARR_AUTHFILE", ""), "`path` to a file containing username:password. Takes precedence over --auth (or YARR_AUTH)")
+	flag.StringVar(&auth, "auth", opt("YARR_AUTH", ""), "string with username and password in the format `username:password`")
+	flag.StringVar(&certfile, "cert-file", opt("YARR_CERTFILE", ""), "`path` to cert file for https")
+	flag.StringVar(&keyfile, "key-file", opt("YARR_KEYFILE", ""), "`path` to key file for https")
+	flag.StringVar(&db, "db", opt("YARR_DB", ""), "storage file `path`")
+	flag.StringVar(&logfile, "log-file", opt("YARR_LOGFILE", ""), "`path` to log file to use instead of stdout")
+	flag.BoolVar(&ver, "version", false, "print application version")
+	flag.BoolVar(&open, "open", false, "open the server in browser")
+	flag.Parse()
+
+	if ver {
+		fmt.Printf("v%s (%s)\n", Version, GitHash)
+		return
+	}
+
+	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
+	if logfile != "" {
+		file, err := os.OpenFile(logfile, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+		if err != nil {
+			log.Fatal("Failed to setup log file: ", err)
+		}
+		defer file.Close()
+		log.SetOutput(file)
+	} else {
+		log.SetOutput(os.Stdout)
+	}
+
+	if db == "" {
+		configPath, err := os.UserConfigDir()
+		if err != nil {
+			log.Fatal("Failed to get config dir: ", err)
+		}
+
+		storagePath := filepath.Join(configPath, "yarr")
+		if err := os.MkdirAll(storagePath, 0755); err != nil {
+			log.Fatal("Failed to create app config dir: ", err)
+		}
+		db = filepath.Join(storagePath, "storage.db")
+	}
+
+	log.Printf("using db file %s", db)
+
+	var username, password string
+	var err error
+	if authfile != "" {
+		f, err := os.Open(authfile)
+		if err != nil {
+			log.Fatal("Failed to open auth file: ", err)
+		}
+		defer f.Close()
+		username, password, err = parseAuthfile(f)
+		if err != nil {
+			log.Fatal("Failed to parse auth file: ", err)
+		}
+	} else if auth != "" {
+		username, password, err = parseAuthfile(strings.NewReader(auth))
+		if err != nil {
+			log.Fatal("Failed to parse auth literal: ", err)
+		}
+	}
+
+	if (certfile != "" || keyfile != "") && (certfile == "" || keyfile == "") {
+		log.Fatalf("Both cert & key files are required")
+	}
+
+	store, err := storage.New(db)
+	if err != nil {
+		log.Fatal("Failed to initialise database: ", err)
+	}
+
+	worker.SetVersion(Version)
+	srv := server.NewServer(store, addr)
+
+	if basepath != "" {
+		srv.BasePath = "/" + strings.Trim(basepath, "/")
+	}
+
+	if certfile != "" && keyfile != "" {
+		srv.CertFile = certfile
+		srv.KeyFile = keyfile
+	}
+
+	if username != "" && password != "" {
+		srv.Username = username
+		srv.Password = password
+	}
+
+	log.Printf("starting server at %s", srv.GetAddr())
+	if open {
+		platform.Open(srv.GetAddr())
+	}
+	platform.Start(srv)
+}

cmd/yarr/main_test.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestPasswordFromAuthfile(t *testing.T) {
+	for _, tc := range [...]struct {
+		authfile         string
+		expectedUsername string
+		expectedPassword string
+		expectedError    bool
+	}{
+		{
+			authfile:         "username:password",
+			expectedUsername: "username",
+			expectedPassword: "password",
+			expectedError:    false,
+		},
+		{
+			authfile:      "username-and-no-password",
+			expectedError: true,
+		},
+		{
+			authfile:         "username:password:with:columns",
+			expectedUsername: "username",
+			expectedPassword: "password:with:columns",
+			expectedError:    false,
+		},
+	} {
+		t.Run(tc.authfile, func(t *testing.T) {
+			username, password, err := parseAuthfile(strings.NewReader(tc.authfile))
+			if tc.expectedUsername != username {
+				t.Errorf("expected username %q, got %q", tc.expectedUsername, username)
+			}
+			if tc.expectedPassword != password {
+				t.Errorf("expected password %q, got %q", tc.expectedPassword, password)
+			}
+			if tc.expectedError && err == nil {
+				t.Errorf("expected error, got nil")
+			} else if !tc.expectedError && err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}

doc/build.md

@@ -0,0 +1,56 @@
+## Compilation
+
+Prerequisies:
+
+* Go >= 1.18
+* C Compiler (GCC / Clang / ...)
+* Zig >= 0.14.0 (optional, for cross-compiling CLI versions)
+* binutils (optional, for building Windows GUI version)
+
+Get the source code:
+
+    git clone https://github.com/nkanaev/yarr.git
+
+Compile:
+
+    # create cli for the host OS/architecture
+    make host               # out/yarr
+
+    # create GUI, works only in the target OS
+    make windows_amd64_gui  # out/windows_amd64_gui/yarr.exe
+    make windows_arm64_gui  # out/windows_arm64_gui/yarr.exe
+    make darwin_arm64_gui   # out/darwin_arm64_gui/yarr.app
+    make darwin_amd64_gui   # out/darwin_amd64_gui/yarr.app
+
+    # create cli, cross-compiles within any OS/architecture
+    make linux_amd64
+    make linux_arm64
+    make linux_armv7
+    make windows_amd64
+    make windows_arm64
+
+    # ... or build a docker image
+    docker build -t yarr -f etc/dockerfile .
+
+## ARM compilation
+
+The instructions below are to cross-compile *yarr* to `Linux/ARM*`.
+
+Build:
+
+    docker build -t yarr.arm -f etc/dockerfile.arm .
+
+Test:
+
+    # inside host
+    docker run -it --rm yarr.arm
+
+    # then, inside container
+    cd /root/out
+    qemu-aarch64 -L /usr/aarch64-linux-gnu/ yarr.arm64
+
+Extract files from images:
+
+    CID=$(docker create yarr.arm)
+    docker cp -a "$CID:/root/out" .
+    docker rm "$CID"

doc/changelog.txt

@@ -1,3 +1,69 @@
+# upcoming 
+
+- (new) Fever API support (thanks to @icefed)
+- (new) editable feed link (thanks to @adaszko)
+- (new) switch to feed by clicking the title in the article page (thanks to @tarasglek for suggestion)
+- (new) support multiple media links
+- (fix) duplicate articles caused by the same feed addition (thanks to @adaszko)
+- (fix) relative article links (thanks to @adazsko for the report)
+- (fix) atom article links stored in id element (thanks to @adazsko for the report)
+- (fix) parsing atom feed titles (thanks to @wnh)
+- (fix) sorting same-day batch articles (thanks to @lamescholar for the report)
+- (fix) showing login page in the selected theme (thanks to @feddiriko for the report)
+- (fix) parsing atom feeds with html elements (thanks to @tillcash & @toBeOfUse for the report, @krkk for the fix)
+- (fix) parsing feeds with missing guids (thanks to @hoyii for the report)
+- (fix) sending actual client version to servers (thanks to @aidanholm)
+- (fix) error caused by missing config dir (thanks to @timster)
+- (etc) load external images with no-referrer policy (thanks to @tillcash for the report)
+- (etc) open external links with no-referrer policy (thanks to @donovanglover)
+- (etc) show article content in the list if title is missing (thanks to @asimpson for suggestion)
+
+# v2.4 (2023-08-15)
+
+- (new) ARM build support (thanks to @tillcash & @fenuks)
+- (new) auth configuration via param or env variable (thanks to @pierreprinetti)
+- (new) web app manifest for an app-like experience on mobile (thanks to @qbit)
+- (fix) concurrency issue crashing the app (thanks to @quoing)
+- (fix) favicon visibility in dark mode (thanks to @caycaycarly for the report)
+- (fix) autoloading more articles not working in certain edge cases (thanks to @fenuks for the report)
+- (fix) handle Google URL redirects in "Read Here" (thanks to @cubbei for discovery)
+- (fix) handle failures to extract content in "Read Here" (thanks to @grigio for the report)
+- (fix) article view width for high resolution screens (thanks to @whaler-ragweed for the report)
+- (fix) make newly added feed searchable (thanks to @BMorearty for the report)
+- (fix) feed/article selection accessibility via arrow keys (thanks to @grigio and @tillcash)
+- (fix) keyboard shortcuts in Firefox (thanks to @kaloyan13)
+- (fix) keyboard shortcuts in non-English layouts (thanks to @kaloyan13)
+- (fix) sorting articles with timezone information (thanks to @x2cf)
+- (fix) handling links set in guid only for certain feeds (thanks to @adaszko for the report)
+- (fix) crashes caused by feed icon endpoint (thanks to @adaszko)
+
+# v2.3 (2022-05-03)
+
+- (fix) handling encodings (thanks to @f100024 & @fserb)
+- (fix) parsing xml feeds with illegal characters (thanks to @stepelu for the report)
+- (fix) old articles reappearing as unread (thanks to @adaszko for the report)
+- (fix) item list scrolling issue on large screens (thanks to @bielej for the report)
+- (fix) keyboard shortcuts color in dark mode (thanks to @John09f9 for the report)
+- (etc) autofocus when adding a new feed (thanks to @lakuapik)
+
+# v2.2 (2021-11-20)
+
+- (fix) windows console support (thanks to @dufferzafar for the report)
+- (fix) remove html tags from article titles (thanks to Alex Went for the report)
+- (etc) autoselect current folder when adding a new feed (thanks to @krkk)
+- (etc) folder/feed settings menu available across all filters
+
+# v2.1 (2021-08-16)
+
+- (new) configuration via env variables
+- (fix) missing `content-type` headers (thanks to @verahawk for the report)
+- (fix) handle opml files not following the spec (thanks to @huangnauh for the report)
+- (fix) pagination in unread/starred feeds (thanks to @Farow for the report)
+- (fix) handling feeds with non-utf8 encodings (thanks to @fserb for the report)
+- (fix) errors caused by empty feeds (thanks to @decke)
+- (fix) recognize all audio mime types as podcasts (thanks to @krkk)
+- (fix) ui tweaks (thanks to @Farow)
+
 # v2.0 (2021-04-18)
 
 - (new) user interface tweaks

doc/fever.md

@@ -0,0 +1,19 @@
+# Fever API support
+
+Fever API is a kind of RSS HTTP API interface, because the Fever API definition is not very clear, so the implementation of Fever server and Client may have some compatibility problems.
+
+The Fever API implemented by Yarr is based on the Fever API spec: https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md.
+
+Here are some Apps that have been tested to work with yarr.  Feel free to test other Clients/Apps and update the list here.
+
+>  Different apps support different URL/Address formats.  Please note whether the URL entered has `http://` scheme and `/` suffix.
+
+| App                                                                       | Platforms        | Config Server URL                                   |
+|:------------------------------------------------------------------------- | ---------------- |:--------------------------------------------------- |
+| [Reeder](https://reederapp.com/)                                          | MacOS<br>iOS     | 127.0.0.1:7070/fever<br>http://127.0.0.1:7070/fever |
+| [ReadKit](https://readkit.app/)                                           | MacOS<br>iOS     | http://127.0.0.1:7070/fever                         |
+| [Fluent Reader](https://github.com/yang991178/fluent-reader)              | MacOS<br>Windows | http://127.0.0.1:7070/fever/                        |
+| [Unread](https://apps.apple.com/us/app/unread-an-rss-reader/id1363637349) | iOS              | http://127.0.0.1:7070/fever                         |
+| [Fiery Feeds](https://voidstern.net/fiery-feeds)                          | MacOS<br>iOS     | http://127.0.0.1:7070/fever                         |
+
+If you are having trouble using Fever, please open an issue and @icefed, thanks.

doc/formats.txt

@@ -159,6 +159,7 @@ Delete any from the list in case they drop support of web feeds.
 - medium
 - posthaven
 - reddit
+- substack
 - tumblr
 - vimeo
 - wordpress

doc/samples.yml

@@ -0,0 +1,68 @@
+- site: https://vimeo.com/channels/staffpicks/videos
+  feed: https://vimeo.com/channels/staffpicks/videos/rss
+  tags: [vimeo, image]
+
+- site: https://www.youtube.com/@everyframeapainting/videos
+  feed: https://www.youtube.com/feeds/videos.xml?channel_id=UCjFqcJQXGZ6T6sxyFB-5i6A"
+  tags: [youtube, image]
+
+- site: https://iwdrm.tumblr.com/
+  feed: https://iwdrm.tumblr.com/rss
+  tags: [tumblr, image]
+
+- site: https://falseknees.tumblr.com/
+  feed: https://falseknees.tumblr.com/rss
+  tags: [tumblr, image]
+
+- site: https://accidentallyquadratic.tumblr.com/
+  feed: https://accidentallyquadratic.tumblr.com/rss
+  info: text blog with code sections
+  tags: [tumblr, text, code]
+
+- site: https://www.flickr.com/photos/maratsafin/
+  feed: https://www.flickr.com/services/feeds/photos_public.gne?id=59021497@N07&lang=en-us&format=atom
+  tags: [flickr, image]
+
+- site: https://www.reddit.com/r/comics
+  feed: https://www.reddit.com/r/comics.rss
+  tags: [reddit, image]
+
+- site: https://www.reddit.com/r/AITAH
+  feed: https://www.reddit.com/r/AITAH.rss
+  tags: [reddit, text]
+
+- site: https://idothei.wordpress.com/
+  feed: https://idothei.wordpress.com/feed/
+  tags: [wordpress, text]
+
+- site: https://www.vidarholen.net/contents/blog/
+  feed: https://www.vidarholen.net/contents/blog/?feed=rss2
+  tags: [wordpress, text]
+
+- site: https://blog.posthaven.com/
+  feed: https://blog.posthaven.com/posts.atom
+  tags: [posthaven, text]
+
+- site: https://medium.com/@dailynewsletter
+  feed: https://medium.com/feed/@dailynewsletter
+  tags: [medium, text]
+
+- site: https://thereveal.substack.com/
+  feed: https://thereveal.substack.com/feed
+  tags: [substack, text]
+
+- site: https://tema.livejournal.com/
+  feed: https://tema.livejournal.com/data/rss
+  tags: [livejournal, text]
+
+- site: https://mametter.hatenablog.com/
+  feed: https://mametter.hatenablog.com/feed
+  tags: [hatena, text]
+
+- site: https://juliepowell.blogspot.com/
+  feed: https://juliepowell.blogspot.com/feeds/posts/default
+  tags: [blogger, text]
+
+- site: https://micro.blog/val
+  feed: https://micro.blog/posts/val
+  tags: [json, microblog]

doc/thirdparty.txt

@@ -20,3 +20,8 @@ The licenses are included, and the authorship comments are left intact.
   https://github.com/getlantern/systray (commit:2c0986d) Apache 2.0
 
   removed golog dependency
+
+- fixconsole
+  https://github.com/apenwarr/fixconsole (commit:5a9f648) Apache 2.0
+
+  removed `w32` dependency

dockerfile

@@ -1,12 +0,0 @@
-FROM golang:alpine AS build
-RUN apk add build-base git
-WORKDIR /src
-COPY . .
-RUN make build_linux
-
-FROM alpine:latest
-RUN apk add --no-cache ca-certificates && \
-    update-ca-certificates
-COPY --from=build /src/_output/linux/yarr /usr/local/bin/yarr
-EXPOSE 7070
-CMD ["/usr/local/bin/yarr", "-addr", "0.0.0.0:7070", "-db", "/data/yarr.db"]

etc/dockerfile

@@ -0,0 +1,14 @@
+FROM golang:alpine3.18 AS build
+RUN apk add build-base git
+WORKDIR /src
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg \
+  make host
+
+FROM alpine:latest
+RUN apk add --no-cache ca-certificates && \
+    update-ca-certificates
+COPY --from=build /src/out/yarr /usr/local/bin/yarr
+EXPOSE 7070
+ENTRYPOINT ["/usr/local/bin/yarr"]
+CMD ["-addr", "0.0.0.0:7070", "-db", "/data/yarr.db"]

etc/dockerfile.arm

@@ -0,0 +1,38 @@
+FROM ubuntu:20.04
+
+# Install GCC
+RUN apt update
+RUN apt install -y \
+        wget build-essential \
+        gcc-aarch64-linux-gnu \
+        binutils-aarch64-linux-gnu binutils-aarch64-linux-gnu-dbg \
+        gcc-arm-linux-gnueabihf \
+        binutils-arm-linux-gnueabihf binutils-arm-linux-gnueabihf-dbg
+RUN env DEBIAN_FRONTEND=noninteractive \
+    apt install -y qemu-user qemu-user-static
+
+# Install Golang
+RUN wget --quiet https://go.dev/dl/go1.18.2.linux-amd64.tar.gz && \
+    rm -rf /usr/local/go && \
+    tar -C /usr/local -xzf go1.18.2.linux-amd64.tar.gz
+ENV PATH=$PATH:/usr/local/go/bin
+
+# Copy source code
+WORKDIR /root/src
+RUN mkdir /root/out
+COPY . .
+
+# Build ARM64
+RUN env \
+        CC=aarch64-linux-gnu-gcc \
+        CGO_ENABLED=1 \
+        GOOS=linux GOARCH=arm64 \
+    make host && mv out/yarr /root/out/yarr.arm64
+
+RUN env \
+        CC=arm-linux-gnueabihf-gcc \
+        CGO_ENABLED=1 \
+        GOOS=linux GOARCH=arm GOARM=7 \
+    make host && mv out/yarr /root/out/yarr.armv7
+
+CMD ["/bin/bash"]

etc/install-linux.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+if [[ ! -d "$HOME/.local/share/applications" ]]; then
+  mkdir -p "$HOME/.local/share/applications"
+fi
+
+cat >"$HOME/.local/share/applications/yarr.desktop" <<END
+[Desktop Entry]
+Name=yarr
+Exec=$HOME/.local/bin/yarr -open
+Icon=yarr
+Type=Application
+Categories=Internet;
+END
+
+if [[ ! -d "$HOME/.local/share/icons" ]]; then
+  mkdir -p "$HOME/.local/share/icons"
+fi
+
+cat >"$HOME/.local/share/icons/yarr.svg" <<END
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-anchor-favicon">
+  <circle cx="12" cy="5" r="3" stroke-width="4" stroke="#ffffff"></circle>
+  <line x1="12" y1="22" x2="12" y2="8" stroke-width="4" stroke="#ffffff"></line>
+  <path d="M5 12H2a10 10 0 0 0 20 0h-3" stroke-width="4" stroke="#ffffff"></path>
+
+  <circle cx="12" cy="5" r="3"></circle>
+  <line x1="12" y1="22" x2="12" y2="8"></line>
+  <path d="M5 12H2a10 10 0 0 0 20 0h-3"></path>
+</svg>
+END

etc/macos_package.sh

@@ -0,0 +1,62 @@
+#/bin/sh
+
+set -e
+
+usage() {
+  echo "usage: $0 VERSION path/to/icon.icns path/to/binary output/dir"
+}
+
+if [ $# -eq 0 ]; then
+    usage
+    exit
+fi
+
+VERSION=$1
+ICNFILE=$2
+BINFILE=$3
+OUTPATH=$4
+
+mkdir -p $OUTPATH/yarr.app/Contents/MacOS
+mkdir -p $OUTPATH/yarr.app/Contents/Resources
+
+mv $BINFILE $OUTPATH/yarr.app/Contents/MacOS/yarr
+cp $ICNFILE $OUTPATH/yarr.app/Contents/Resources/icon.icns
+
+chmod u+x  $OUTPATH/yarr.app/Contents/MacOS/yarr
+
+echo -n 'APPL????' >$OUTPATH/yarr.app/Contents/PkgInfo
+cat <<EOF >$OUTPATH/yarr.app/Contents/Info.plist
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleName</key>
+	<string>yarr</string>
+	<key>CFBundleDisplayName</key>
+	<string>yarr</string>
+	<key>CFBundleIdentifier</key>
+	<string>nkanaev.yarr</string>
+	<key>CFBundleVersion</key>
+	<string>$VERSION</string>
+	<key>CFBundlePackageType</key>
+	<string>APPL</string>
+	<key>CFBundleExecutable</key>
+	<string>yarr</string>
+
+	<key>CFBundleIconFile</key>
+	<string>icon</string>
+	<key>LSApplicationCategoryType</key>
+	<string>public.app-category.news</string>
+
+	<key>NSHighResolutionCapable</key>
+	<string>True</string>
+
+	<key>LSMinimumSystemVersion</key>
+	<string>10.13</string>
+	<key>LSUIElement</key>
+	<true/>
+	<key>NSHumanReadableCopyright</key>
+	<string>Copyright © 2020 nkanaev. All rights reserved.</string>
+</dict>
+</plist>
+EOF

etc/windows_versioninfo.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+set -e
+
+# Function to display usage information
+usage() {
+  echo "Usage: $0 [-version VERSION] [-outfile FILENAME]"
+  echo "  -version VERSION   Set the version number (default: 0.0)"
+  echo "  -outfile FILENAME  Set the output file name (default: versioninfo.rc)"
+  echo ""
+  echo "This script generates a Windows resource file with version information."
+  exit 1
+}
+
+# Default values
+version="0.0"
+outfile="versioninfo.rc"
+
+# Check if help is requested
+if [[ "$1" == "-h" || "$1" == "--help" ]]; then
+  usage
+fi
+
+if [ $# -eq 0 ]; then
+  usage
+fi
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -version)
+      if [[ -z "$2" || "$2" == -* ]]; then
+        echo "Error: Missing value for -version parameter"
+        usage
+      fi
+      version="$2"
+      shift 2
+      ;;
+    -outfile)
+      if [[ -z "$2" || "$2" == -* ]]; then
+        echo "Error: Missing value for -outfile parameter"
+        usage
+      fi
+      outfile="$2"
+      shift 2
+      ;;
+    *)
+      echo "Error: Unknown parameter: $1"
+      usage
+      ;;
+  esac
+done
+
+# Replace dots with commas for version_comma
+version_comma="${version//./,}"
+
+# Use a here document for the template with ENDFILE delimiter
+cat <<ENDFILE > "$outfile"
+1 VERSIONINFO
+FILEVERSION     $version_comma,0,0
+PRODUCTVERSION  $version_comma,0,0
+BEGIN
+  BLOCK "StringFileInfo"
+  BEGIN
+    BLOCK "080904E4"
+    BEGIN
+      VALUE "CompanyName", "Old MacDonald's Farm"
+      VALUE "FileDescription", "Yet another RSS reader"
+      VALUE "FileVersion", "$version"
+      VALUE "InternalName", "yarr"
+      VALUE "LegalCopyright", "nkanaev"
+      VALUE "OriginalFilename", "yarr.exe"
+      VALUE "ProductName", "yarr"
+      VALUE "ProductVersion", "$version"
+    END
+  END
+  BLOCK "VarFileInfo"
+  BEGIN
+    VALUE "Translation", 0x809, 1252
+  END
+END
+
+1 ICON "icon.ico"
+ENDFILE
+
+# Set the correct permissions
+chmod 644 "$outfile"
+
+echo "Generated $outfile with version $version"

go.mod

@@ -1,9 +1,11 @@
 module github.com/nkanaev/yarr
 
-go 1.16
+go 1.18
 
 require (
-	github.com/mattn/go-sqlite3 v1.14.0
-	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
-	golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13
+	github.com/mattn/go-sqlite3 v1.14.7
+	golang.org/x/net v0.36.0
+	golang.org/x/sys v0.30.0
 )
+
+require golang.org/x/text v0.22.0 // indirect

go.sum

@@ -1,15 +1,8 @@
-github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
-github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
-github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13 h1:5jaG59Zhd+8ZXe8C+lgiAGqkOaZBruqrWclLkgAww34=
-golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+github.com/mattn/go-sqlite3 v1.14.7 h1:fxWBnXkxfM6sRiuH3bqJ4CfzZojMOLVc0UTsTglEghA=
+github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA=
+golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=

makefile

@@ -1,39 +1,89 @@
-VERSION=2.0
+VERSION=2.4
 GITHASH=$(shell git rev-parse --short=8 HEAD)
 
-CGO_ENABLED=1
+GO_TAGS    = sqlite_foreign_keys sqlite_json
+GO_LDFLAGS = -s -w -X 'main.Version=$(VERSION)' -X 'main.GitHash=$(GITHASH)'
 
-GO_LDFLAGS  = -s -w
-GO_LDFLAGS := $(GO_LDFLAGS) -X 'main.Version=$(VERSION)' -X 'main.GitHash=$(GITHASH)'
+GO_FLAGS         = -tags "$(GO_TAGS)"     -ldflags="$(GO_LDFLAGS)"
+GO_FLAGS_GUI     = -tags "$(GO_TAGS) gui" -ldflags="$(GO_LDFLAGS)"
+GO_FLAGS_GUI_WIN = -tags "$(GO_TAGS) gui" -ldflags="$(GO_LDFLAGS) -H windowsgui"
 
-build_default:
-	mkdir -p _output
-	go build -tags "sqlite_foreign_keys release" -ldflags="$(GO_LDFLAGS)" -o _output/yarr src/main.go
+export CGO_ENABLED=1
 
-build_macos:
-	set GOOS=darwin
-	set GOARCH=amd64
-	mkdir -p _output/macos
-	go build -tags "sqlite_foreign_keys release macos" -ldflags="$(GO_LDFLAGS)" -o _output/macos/yarr src/main.go
-	cp src/platform/icon.png _output/macos/icon.png
-	go run bin/package_macos.go -outdir _output/macos -version "$(VERSION)"
+default: test host
 
-build_linux:
-	set GOOS=linux
-	set GOARCH=386
-	mkdir -p _output/linux
-	go build -tags "sqlite_foreign_keys release linux" -ldflags="$(GO_LDFLAGS)" -o _output/linux/yarr src/main.go
+# platform-specific files
 
-build_windows:
-	set GOOS=windows
-	set GOARCH=386
-	mkdir -p _output/windows
-	go run bin/generate_versioninfo.go -version "$(VERSION)" -outfile src/platform/versioninfo.rc
+etc/icon.icns: etc/icon_macos.png
+	mkdir -p etc/icon.iconset
+	sips -s format png --resampleWidth 1024 etc/icon_macos.png --out etc/icon.iconset/icon_512x512@2x.png
+	sips -s format png --resampleWidth  512 etc/icon_macos.png --out etc/icon.iconset/icon_512x512.png
+	sips -s format png --resampleWidth  256 etc/icon_macos.png --out etc/icon.iconset/icon_256x256.png
+	sips -s format png --resampleWidth  128 etc/icon_macos.png --out etc/icon.iconset/icon_128x128.png
+	sips -s format png --resampleWidth   64 etc/icon_macos.png --out etc/icon.iconset/icon_32x32@2x.png
+	sips -s format png --resampleWidth   32 etc/icon_macos.png --out etc/icon.iconset/icon_32x32.png
+	sips -s format png --resampleWidth   16 etc/icon_macos.png --out etc/icon.iconset/icon_16x16.png
+	iconutil -c icns etc/icon.iconset -o etc/icon.icns
+
+src/platform/versioninfo.rc:
+	./etc/windows_versioninfo.sh -version "$(VERSION)" -outfile src/platform/versioninfo.rc
 	windres -i src/platform/versioninfo.rc -O coff -o src/platform/versioninfo.syso
-	go build -tags "sqlite_foreign_keys release windows" -ldflags="$(GO_LDFLAGS) -H windowsgui" -o _output/windows/yarr.exe src/main.go
+
+# build targets
+
+host:
+	go build $(GO_FLAGS) -o out/yarr ./cmd/yarr
+
+darwin_amd64:
+	# cross-compilation not supported: CC="zig cc -target x86_64-macos-none"
+	GOOS=darwin GOARCH=arm64 go build $(GO_FLAGS) -o out/$@/yarr ./cmd/yarr
+
+darwin_arm64:
+	# cross-compilation not supported: CC="zig cc -target aarch64-macos-none"
+	GOOS=darwin GOARCH=arm64 go build $(GO_FLAGS) -o out/$@/yarr ./cmd/yarr
+
+linux_amd64:
+	CC="zig cc -target x86_64-linux-musl -O2 -g0" CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux GOARCH=amd64 \
+	go build $(GO_FLAGS) -o out/$@/yarr ./cmd/yarr
+
+linux_arm64:
+	CC="zig cc -target aarch64-linux-musl -O2 -g0" CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux GOARCH=arm64 \
+	go build $(GO_FLAGS) -o out/$@/yarr ./cmd/yarr
+
+linux_armv7:
+	CC="zig cc -target arm-linux-musleabihf -O2 -g0" CGO_CFLAGS="-D_LARGEFILE64_SOURCE" GOOS=linux GOARCH=arm GOARM=7 \
+	go build $(GO_FLAGS) -o out/$@/yarr ./cmd/yarr
+
+windows_amd64:
+	CC="zig cc -target x86_64-windows-gnu" GOOS=windows GOARCH=amd64 go build $(GO_FLAGS) -o out/$@/yarr ./cmd/yarr
+
+windows_arm64:
+	CC="zig cc -target aarch64-windows-gnu" GOOS=windows GOARCH=arm64 go build $(GO_FLAGS) -o out/$@/yarr ./cmd/yarr
+
+darwin_arm64_gui: etc/icon.icns
+	GOOS=darwin GOARCH=arm64 go build $(GO_FLAGS_GUI) -o out/$@/yarr ./cmd/yarr
+	./etc/macos_package.sh $(VERSION) etc/icon.icns out/$@/yarr out/$@
+
+darwin_amd64_gui: etc/icon.icns
+	GOOS=darwin GOARCH=amd64 go build $(GO_FLAGS_GUI) -o out/$@/yarr ./cmd/yarr
+	./etc/macos_package.sh $(VERSION) etc/icon.icns out/$@/yarr out/$@
+
+windows_amd64_gui: src/platform/versioninfo.rc
+	GOOS=windows GOARCH=amd64 go build $(GO_FLAGS_GUI_WIN) -o out/$@/yarr.exe ./cmd/yarr
+
+windows_arm64_gui: src/platform/versioninfo.rc
+	GOOS=windows GOARCH=arm64 go build $(GO_FLAGS_GUI_WIN) -o out/$@/yarr.exe ./cmd/yarr
 
 serve:
-	go run -tags "sqlite_foreign_keys" src/main.go -db local.db
+	go run $(GO_FLAGS) ./cmd/yarr -db local.db
 
 test:
-	cd src && go test -tags "sqlite_foreign_keys release" ./...
+	go test $(GO_FLAGS) ./...
+
+.PHONY: \
+	host \
+	darwin_amd64 darwin_amd64_gui \
+	darwin_arm64 darwin_arm64_gui \
+	windows_amd64 windows_amd64_gui \
+	windows_arm64 windows_arm64_gui \
+	serve test

readme.md

@@ -3,72 +3,36 @@
 **yarr** (yet another rss reader) is a web-based feed aggregator which can be used both
 as a desktop application and a personal self-hosted server.
 
-It is written in Go with the frontend in Vue.js. The storage is backed by SQLite.
+The app is a single binary with an embedded database (SQLite).
 
 ![screenshot](etc/promo.png)
 
 ## usage
 
-The latest prebuilt binaries for Linux/MacOS/Windows are available
-[here](https://github.com/nkanaev/yarr/releases/latest).
+The latest prebuilt binaries for Linux/MacOS/Windows AMD64 are available
+[here](https://github.com/nkanaev/yarr/releases/latest). Installation instructions:
 
-### macos
+* MacOS
 
-Download `yarr-*-macos64.zip`, unzip it, place `yarr.app` in `/Applications` folder.
+  Download `yarr-*-macos64.zip`, unzip it, place `yarr.app` in `/Applications` folder, [open the app][macos-open], click the anchor menu bar icon, select "Open".
 
-The binaries are not signed, because the author doesn't want to buy a certificate.
-Apple hates cheapskate developers, therefore the OS will refuse to run the application.
-To bypass these measures, you can run the command:
+* Windows
 
-    xattr -d com.apple.quarantine /Applications/yarr.app
+  Download `yarr-*-windows64.zip`, unzip it, open `yarr.exe`, click the anchor system tray icon, select "Open".
 
-### windows
+* Linux
 
-Download `yarr-*-windows32.zip`, unzip it, place wherever you'd like to
-(`C:\Program Files` or Recycle Bin). Create a shortcut manually if you'd like to.
+  Download `yarr-*-linux64.zip`, unzip it, place `yarr` in `$HOME/.local/bin`
+and run [the script](etc/install-linux.sh).
 
-Microsoft doesn't like cheapskate developers too,
-but might only gently warn you about that, which you can safely ignore.
-
-### linux
-
-The Linux version doesn't come with the desktop environment integration.
-For easy access on DE it is recommended to create a desktop menu entry by
-by following the steps below:
-
-    unzip -x yarr*.zip
-    sudo mv yarr /usr/local/bin/yarr
-    sudo nano /usr/local/share/applications/yarr.desktop
-
-and pasting the content:
-
-    [Desktop Entry]
-    Name=yarr
-    Exec=/usr/local/bin/yarr -open
-    Icon=rss
-    Type=Application
-    Categories=Internet;
+[macos-open]: https://support.apple.com/en-gb/guide/mac-help/mh40616/mac
 
 For self-hosting, see `yarr -h` for auth, tls & server configuration flags.
 
-## build
+See more:
 
-Install `Go >= 1.16` and `gcc`. Get the source code:
-
-    git clone https://github.com/nkanaev/yarr.git
-
-Then run one of the corresponding commands:
-
-    # create an executable for the host os
-    make build_macos    # -> _output/macos/yarr.app
-    make build_linux    # -> _output/linux/yarr
-    make build_windows  # -> _output/windows/yarr.exe
-
-    # ... or start a dev server locally
-    make serve          # starts a server at http://localhost:7070
-
-    # ... or build a docker image
-    docker build -t yarr .
+* [Building from source code](doc/build.md)
+* [Fever API support](doc/fever.md)
 
 ## credits
 

src/assets/assets.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/fs"
 	"io/ioutil"
+	"log"
 	"os"
 )
 
@@ -29,9 +30,18 @@ func Template(path string) *template.Template {
 	if !found {
 		tmpl = template.Must(template.New(path).Delims("{%", "%}").Funcs(template.FuncMap{
 			"inline": func(svg string) template.HTML {
-				svgfile, _ := FS.Open("graphicarts/" + svg)
-				content, _ := ioutil.ReadAll(svgfile)
-				svgfile.Close()
+				svgfile, err := FS.Open("graphicarts/" + svg)
+				// should never happen
+				if err != nil {
+					log.Fatal(err)
+				}
+				defer svgfile.Close()
+
+				content, err := ioutil.ReadAll(svgfile)
+				// should never happen
+				if err != nil {
+					log.Fatal(err)
+				}
 				return template.HTML(content)
 			},
 		}).ParseFS(FS, path))

src/assets/assetsfs.go

@@ -1,5 +1,3 @@
-// +build release
-
 package assets
 
 import "embed"

src/assets/graphicarts/chevron-down.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-chevron-down"><polyline points="6 9 12 15 18 9"></polyline></svg>

src/assets/graphicarts/favicon.svg

@@ -0,0 +1,9 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-anchor-favicon">
+  <circle cx="12" cy="5" r="3" stroke-width="4" stroke="#ffffff"></circle>
+  <line x1="12" y1="22" x2="12" y2="8" stroke-width="4" stroke="#ffffff"></line>
+  <path d="M5 12H2a10 10 0 0 0 20 0h-3" stroke-width="4" stroke="#ffffff"></path>
+
+  <circle cx="12" cy="5" r="3"></circle>
+  <line x1="12" y1="22" x2="12" y2="8"></line>
+  <path d="M5 12H2a10 10 0 0 0 20 0h-3"></path>
+</svg>

src/assets/graphicarts/list.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-list"><line x1="8" y1="6" x2="21" y2="6"></line><line x1="8" y1="12" x2="21" y2="12"></line><line x1="8" y1="18" x2="21" y2="18"></line><line x1="3" y1="6" x2="3.01" y2="6"></line><line x1="3" y1="12" x2="3.01" y2="12"></line><line x1="3" y1="18" x2="3.01" y2="18"></line></svg>

src/assets/graphicarts/menu.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-menu"><line x1="3" y1="12" x2="21" y2="12"></line><line x1="3" y1="6" x2="21" y2="6"></line><line x1="3" y1="18" x2="21" y2="18"></line></svg>

src/assets/graphicarts/more-vertical.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-more-vertical"><circle cx="12" cy="12" r="1"></circle><circle cx="12" cy="5" r="1"></circle><circle cx="12" cy="19" r="1"></circle></svg>

src/assets/graphicarts/settings.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-settings"><circle cx="12" cy="12" r="3"></circle><path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"></path></svg>

src/assets/graphicarts/trash-2.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-trash-2"><polyline points="3 6 5 6 21 6"></polyline><path d="M19 6v14a2 2 0 0 1-2 2H7a2 2 0 0 1-2-2V6m3 0V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"></path><line x1="10" y1="11" x2="10" y2="17"></line><line x1="14" y1="11" x2="14" y2="17"></line></svg>

src/assets/index.html

@@ -5,7 +5,9 @@
     <title>yarr!</title>
     <link rel="stylesheet" href="./static/stylesheets/bootstrap.min.css">
     <link rel="stylesheet" href="./static/stylesheets/app.css">
-    <link rel="icon shortcut" href="./static/graphicarts/icon.png">
+    <link rel="icon" href="./static/graphicarts/favicon.svg" type="image/svg+xml">
+    <link rel="alternate icon" href="./static/graphicarts/favicon.png" type="image/png">
+    <link rel="manifest" href="./manifest.json" />
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
     <script>
         window.app = window.app || {}
@@ -23,18 +25,21 @@
                 <div class="flex-grow-1"></div>
                 <button class="toolbar-item"
                         :class="{active: filterSelected == 'unread'}"
+                        :aria-pressed="filterSelected == 'unread'"
                         title="Unread"
                         @click="filterSelected = 'unread'">
                     <span class="icon">{% inline "circle-full.svg" %}</span>
                 </button>
                 <button class="toolbar-item"
                         :class="{active: filterSelected == 'starred'}"
+                        :aria-pressed="filterSelected == 'starred'"
                         title="Starred"
                         @click="filterSelected = 'starred'">
                     <span class="icon">{% inline "star-full.svg" %}</span>
                 </button>
                 <button class="toolbar-item"
                         :class="{active: filterSelected == ''}"
+                        :aria-pressed="filterSelected == ''"
                         title="All"
                         @click="filterSelected = ''">
                     <span class="icon">{% inline "assorted.svg" %}</span>
@@ -57,25 +62,39 @@
 
                     <div class="dropdown-divider"></div>
 
-                    <header class="dropdown-header">Auto Refresh</header>
+                    <header class="dropdown-header" role="heading" aria-level="2">Theme</header>
                     <div class="row text-center m-0">
-                        <button class="dropdown-item col-4 px-0" :class="{active: !refreshRate}" @click.stop="refreshRate = 0">0</button>
-                        <button class="dropdown-item col-4 px-0" :class="{active: refreshRate == 10}" @click.stop="refreshRate = 10">10m</button>
-                        <button class="dropdown-item col-4 px-0" :class="{active: refreshRate == 30}" @click.stop="refreshRate = 30">30m</button>
-                        <button class="dropdown-item col-4 px-0" :class="{active: refreshRate == 60}" @click.stop="refreshRate = 60">1h</button>
-                        <button class="dropdown-item col-4 px-0" :class="{active: refreshRate == 120}" @click.stop="refreshRate = 120">2h</button>
-                        <button class="dropdown-item col-4 px-0" :class="{active: refreshRate == 240}" @click.stop="refreshRate = 240">4h</button>
+                        <button class="btn btn-link col-4 px-0 rounded-0"
+                                :class="'theme-'+t"
+                                :aria-label="t"
+                                :aria-pressed="theme.name == t"
+                                @click.stop="theme.name = t"
+                                v-for="t in ['light', 'sepia', 'night']">
+                            <span class="icon" v-if="theme.name == t">{% inline "check.svg" %}</span>
+                        </button>
                     </div>
 
                     <div class="dropdown-divider"></div>
 
-                    <header class="dropdown-header">Show first</header>
+                    <header class="dropdown-header" role="heading" aria-level="2">Auto Refresh</header>
+                    <div class="row text-center m-0">
+                        <button class="dropdown-item col-4 px-0" :aria-pressed="!refreshRate"       :class="{active: !refreshRate}"       @click.stop="refreshRate = 0">0</button>
+                        <button class="dropdown-item col-4 px-0" :aria-pressed="refreshRate == 10"  :class="{active: refreshRate == 10}"  @click.stop="refreshRate = 10">10m</button>
+                        <button class="dropdown-item col-4 px-0" :aria-pressed="refreshRate == 30"  :class="{active: refreshRate == 30}"  @click.stop="refreshRate = 30">30m</button>
+                        <button class="dropdown-item col-4 px-0" :aria-pressed="refreshRate == 60"  :class="{active: refreshRate == 60}"  @click.stop="refreshRate = 60">1h</button>
+                        <button class="dropdown-item col-4 px-0" :aria-pressed="refreshRate == 120" :class="{active: refreshRate == 120}" @click.stop="refreshRate = 120">2h</button>
+                        <button class="dropdown-item col-4 px-0" :aria-pressed="refreshRate == 240" :class="{active: refreshRate == 240}" @click.stop="refreshRate = 240">4h</button>
+                    </div>
+
+                    <div class="dropdown-divider"></div>
+
+                    <header class="dropdown-header" role="heading" aria-level="2">Show first</header>
                     <div class="d-flex text-center">
-                        <button class="dropdown-item px-0" :class="{active: itemSortNewestFirst}" @click.stop="itemSortNewestFirst=true">New</button>
-                        <button class="dropdown-item px-0" :class="{active: !itemSortNewestFirst}" @click.stop="itemSortNewestFirst=false">Old</button>
+                        <button class="dropdown-item px-0" :aria-pressed="itemSortNewestFirst"  :class="{active: itemSortNewestFirst}"  @click.stop="itemSortNewestFirst=true">New</button>
+                        <button class="dropdown-item px-0" :aria-pressed="!itemSortNewestFirst" :class="{active: !itemSortNewestFirst}" @click.stop="itemSortNewestFirst=false">Old</button>
                     </div>
                     <div class="dropdown-divider"></div>
-                    <header class="dropdown-header">Subscriptions</header>
+                    <header class="dropdown-header" role="heading" aria-level="2">Subscriptions</header>
                     <form id="opml-import-form" enctype="multipart/form-data" tabindex="-1">
                         <input type="file"
                                id="opml-import"
@@ -117,9 +136,10 @@
                 <div v-for="folder in foldersWithFeeds">
                     <label class="selectgroup mt-1"
                            :class="{'d-none': filterSelected
+                                              && !(current.folder.id == folder.id || current.feed.folder_id == folder.id)
                                               && !filteredFolderStats[folder.id]
-                                              && (!itemSelectedDetails || feedsById[itemSelectedDetails.feed_id].folder_id != folder.id)}">
-                        <input type="radio" name="feed" :value="'folder:'+folder.id" v-model="feedSelected">
+                                              && (!itemSelectedDetails || (feedsById[itemSelectedDetails.feed_id] || {}).folder_id != folder.id)}">
+                        <input type="radio" name="feed" :value="'folder:'+folder.id" v-model="feedSelected" v-if="folder.id">
                         <div class="selectgroup-label d-flex align-items-center w-100" v-if="folder.id">
                             <span class="icon mr-2"
                                   :class="{expanded: folder.is_expanded}"
@@ -133,6 +153,7 @@
                     <div v-show="!folder.id || folder.is_expanded" class="mt-1" :class="{'pl-3': folder.id}">
                         <label class="selectgroup"
                               :class="{'d-none': filterSelected
+                                                 && !(current.feed.id == feed.id)
                                                  && !filteredFeedStats[feed.id]
                                                  && (!itemSelectedDetails || itemSelectedDetails.feed_id != feed.id)}"
                                v-for="feed in folder.feeds">
@@ -177,20 +198,25 @@
                         title="Mark All Read">
                     <span class="icon">{% inline "check.svg" %}</span>
                 </button>
+
+
+                <button class="btn btn-link toolbar-item px-2 ml-2" v-if="!current.type" disabled>
+                    <span class="icon">{% inline "more-horizontal.svg" %}</span>
+                </button>
                 <dropdown class="settings-dropdown"
                           toggle-class="btn btn-link toolbar-item px-2 ml-2"
                           drop="right"
                           title="Feed Settings"
-                          v-if="!filterSelected && current.type == 'feed'">
+                          v-if="current.type == 'feed'">
                     <template v-slot:button>
                         <span class="icon">{% inline "more-horizontal.svg" %}</span>
                     </template>
-                    <header class="dropdown-header">{{ current.feed.title }}</header>
-                    <a class="dropdown-item" :href="current.feed.link" target="_blank" v-if="current.feed.link">
+                    <header class="dropdown-header" role="heading" aria-level="2">{{ current.feed.title }}</header>
+                    <a class="dropdown-item" :href="current.feed.link" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer" v-if="current.feed.link">
                         <span class="icon mr-1">{% inline "globe.svg" %}</span>
                         Website
                     </a>
-                    <a class="dropdown-item" :href="current.feed.feed_link" target="_blank" v-if="current.feed.feed_link">
+                    <a class="dropdown-item" :href="current.feed.feed_link" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer" v-if="current.feed.feed_link">
                         <span class="icon mr-1">{% inline "rss.svg" %}</span>
                         Feed Link
                     </a>
@@ -199,8 +225,12 @@
                         <span class="icon mr-1">{% inline "edit.svg" %}</span>
                         Rename
                     </button>
+                    <button class="dropdown-item" @click="updateFeedLink(current.feed)" v-if="current.feed.feed_link">
+                        <span class="icon mr-1">{% inline "edit.svg" %}</span>
+                        Change Link
+                    </button>
                     <div class="dropdown-divider"></div>
-                    <header class="dropdown-header">Move to...</header>
+                    <header class="dropdown-header" role="heading" aria-level="2">Move to...</header>
                     <button class="dropdown-item"
                         v-if="folder.id != current.feed.folder_id"
                         v-for="folder in folders"
@@ -226,11 +256,11 @@
                           toggle-class="btn btn-link toolbar-item px-2 ml-2"
                           title="Folder Settings"
                           drop="right"
-                          v-if="!filterSelected && current.type == 'folder'">
+                          v-if="current.type == 'folder'">
                     <template v-slot:button>
                         <span class="icon">{% inline "more-horizontal.svg" %}</span>
                     </template>
-                    <header class="dropdown-header">{{ current.folder.title }}</header>
+                    <header class="dropdown-header" role="heading" aria-level="2">{{ current.folder.title }}</header>
                     <button class="dropdown-item" @click="renameFolder(current.folder)">
                         <span class="icon mr-1">{% inline "edit.svg" %}</span>
                         Rename
@@ -253,14 +283,14 @@
                                 <span class="icon icon-small mr-1" v-if="item.status=='starred'">{% inline "star-full.svg" %}</span>
                             </transition>
                             <small class="flex-fill text-truncate mr-1">
-                                {{ feedsById[item.feed_id].title }}
+                                {{ (feedsById[item.feed_id] || {}).title }}
                             </small>
-                            <small class="flex-shrink-0"><relative-time :val="item.date"/></small>
+                            <small class="flex-shrink-0"><relative-time v-bind:title="formatDate(item.date)" :val="item.date"/></small>
                         </div>
                         <div>{{ item.title || 'untitled' }}</div>
                     </div>
                 </label>
-                <button class="btn btn-link btn-block loading my-3" v-if="itemsPage.cur < itemsPage.num"></button>
+                <button class="btn btn-link btn-block loading my-3" v-if="itemsHasMore"></button>
             </div>
             <div class="px-3 py-2 border-top text-danger text-break" v-if="feed_errors[current.feed.id]">
                 {{ feed_errors[current.feed.id] }}
@@ -285,14 +315,6 @@
                     <template v-slot:button>
                         <span class="icon">{% inline "sliders.svg" %}</span>
                     </template>
-                    <div class="row text-center m-0">
-                        <button class="btn btn-link col-4 px-0 rounded-0"
-                                :class="'theme-'+t"
-                                @click.stop="theme.name = t"
-                                v-for="t in ['light', 'sepia', 'night']">
-                            <span class="icon" v-if="theme.name == t">{% inline "check.svg" %}</span>
-                        </button>
-                    </div>
 
                     <button class="dropdown-item" :class="{active: !theme.font}" @click.stop="theme.font = ''">sans-serif</button>
                     <button class="dropdown-item font-serif" :class="{active: theme.font == 'serif'}" @click.stop="theme.font = 'serif'">serif</button>
@@ -309,10 +331,16 @@
                         title="Read Here">
                     <span class="icon" :class="{'icon-loading': loading.readability}">{% inline "book-open.svg" %}</span>
                 </button>
-                <a class="toolbar-item" :href="itemSelectedDetails.link" target="_blank" title="Open Link">
+                <a class="toolbar-item" :href="itemSelectedDetails.link" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer" title="Open Link">
                     <span class="icon">{% inline "external-link.svg" %}</span>
                 </a>
                 <div class="flex-grow-1"></div>
+                <button class="toolbar-item" @click="navigateToItem(-1)" title="Previous Article" :disabled="itemSelected == items[0].id">
+                    <span class="icon">{% inline "chevron-left.svg" %}</span>
+                </button>
+                <button class="toolbar-item" @click="navigateToItem(+1)" title="Next Article" :disabled="itemSelected == items[items.length - 1].id">
+                    <span class="icon">{% inline "chevron-right.svg" %}</span>
+                </button>
                 <button class="toolbar-item" @click="itemSelected=null" title="Close Article">
                     <span class="icon">{% inline "x.svg" %}</span>
                 </button>
@@ -322,17 +350,29 @@
                  class="content px-4 pt-3 pb-5 border-top overflow-auto"
                  :class="{'font-serif': theme.font == 'serif', 'font-monospace': theme.font == 'monospace'}"
                  :style="{'font-size': theme.size + 'rem'}">
-                <h1><b>{{ itemSelectedDetails.title || 'untitled' }}</b></h1>
-                <div class="text-muted">
-                    <div>{{ feedsById[itemSelectedDetails.feed_id].title }}</div>
-                    <time>{{ formatDate(itemSelectedDetails.date) }}</time>
+                <div class="content-wrapper">
+                    <h1><b>{{ itemSelectedDetails.title || 'untitled' }}</b></h1>
+                    <div class="text-muted">
+                        <div>
+                            <span class="cursor-pointer" @click="feedSelected = 'feed:'+(feedsById[itemSelectedDetails.feed_id] || {}).id">
+                                {{ (feedsById[itemSelectedDetails.feed_id] || {}).title }}
+                            </span>
+                        </div>
+                        <time>{{ formatDate(itemSelectedDetails.date) }}</time>
+                    </div>
+                    <hr>
+                    <div v-if="!itemSelectedReadability">
+                        <div v-if="contentImages.length">
+                            <figure v-for="media in contentImages">
+                                <img :src="media.url" loading="lazy">
+                                <figcaption v-if="media.description" v-html="media.description"></figcaption>
+                            </figure>
+                        </div>
+                        <audio class="w-100" controls v-for="media in contentAudios" :src="media.url"></audio>
+                        <video class="w-100" controls v-for="media in contentVideos" :src="media.url"></video>
+                    </div>
+                    <div v-html="itemSelectedContent"></div>
                 </div>
-                <hr>
-                <div v-if="!itemSelectedReadability">
-                    <img :src="itemSelectedDetails.image" v-if="itemSelectedDetails.image" class="mb-3">
-                    <audio class="w-100" controls v-if="itemSelectedDetails.podcast_url" :src="itemSelectedDetails.podcast_url"></audio>
-                </div>
-                <div v-html="itemSelectedContent"></div>
             </div>
         </div>
         <modal :open="!!settings" @hide="settings = ''">
@@ -343,14 +383,14 @@
                 <p class="cursor-default"><b>New Feed</b></p>
                 <form action="" @submit.prevent="createFeed(event)" class="mt-4">
                     <label for="feed-url">URL</label>
-                    <input id="feed-url" name="url" type="url" class="form-control" required autocomplete="off" :readonly="feedNewChoice.length > 0">
+                    <input id="feed-url" name="url" type="url" class="form-control" required autocomplete="off" :readonly="feedNewChoice.length > 0" placeholder="https://example.com/feed" v-focus>
                     <label for="feed-folder" class="mt-3 d-block">
                         Folder
                         <a href="#" class="float-right text-decoration-none" @click.prevent="createNewFeedFolder()">new folder</a>
                     </label>
                     <select class="form-control" id="feed-folder" name="folder_id" ref="newFeedFolder">
                         <option value="">---</option>
-                        <option :value="folder.id" v-for="folder in folders">{{ folder.title }}</option>
+                        <option :value="folder.id" v-for="folder in folders" :selected="folder.id === current.feed.folder_id || folder.id === current.folder.id">{{ folder.title }}</option>
                     </select>
                     <div class="mt-4" v-if="feedNewChoice.length">
                         <p class="mb-2">

src/assets/javascripts/api.js

@@ -105,7 +105,7 @@
       return api('post', './logout')
     },
     crawl: function(url) {
-      return api('get', './page?url=' + url).then(json)
+      return api('get', './page?url=' + encodeURIComponent(url)).then(json)
     }
   }
 })()

src/assets/javascripts/app.js

@@ -2,6 +2,26 @@
 
 var TITLE = document.title
 
+function scrollto(target, scroll) {
+  var padding = 10
+  var targetRect = target.getBoundingClientRect()
+  var scrollRect = scroll.getBoundingClientRect()
+
+  // target
+  var relativeOffset = targetRect.y - scrollRect.y
+  var absoluteOffset = relativeOffset + scroll.scrollTop
+
+  if (padding <= relativeOffset && relativeOffset + targetRect.height <= scrollRect.height - padding) return
+
+  var newPos = scroll.scrollTop
+  if (relativeOffset < padding) {
+    newPos = absoluteOffset - padding
+  } else {
+    newPos = absoluteOffset - scrollRect.height + targetRect.height + padding
+  }
+  scroll.scrollTop = Math.round(newPos)
+}
+
 var debounce = function(callback, wait) {
   var timeout
   return function() {
@@ -21,6 +41,12 @@ Vue.directive('scroll', {
   },
 })
 
+Vue.directive('focus', {
+  inserted: function(el) {
+    el.focus()
+  }
+})
+
 Vue.component('drag', {
   props: ['width'],
   template: '<div class="drag"></div>',
@@ -47,13 +73,13 @@ Vue.component('drag', {
 })
 
 Vue.component('dropdown', {
-  props: ['class', 'toggle-class', 'ref', 'drop'],
+  props: ['class', 'toggle-class', 'ref', 'drop', 'title'],
   data: function() {
     return {open: false}
   },
   template: `
     <div class="dropdown" :class="$attrs.class">
-      <button ref="btn" @click="toggle" :class="btnToggleClass"><slot name="button"></slot></button>
+      <button ref="btn" @click="toggle" :class="btnToggleClass" :title="$props.title"><slot name="button"></slot></button>
       <div ref="menu" class="dropdown-menu" :class="{show: open}"><slot v-if="open"></slot></div>
     </div>
   `,
@@ -180,7 +206,7 @@ var vm = new Vue({
   created: function() {
     this.refreshStats()
       .then(this.refreshFeeds.bind(this))
-      .then(this.refreshItems.bind(this))
+      .then(this.refreshItems.bind(this, false))
 
     api.feeds.list_errors().then(function(errors) {
       vm.feed_errors = errors
@@ -197,10 +223,7 @@ var vm = new Vue({
       'feedNewChoice': [],
       'feedNewChoiceSelected': '',
       'items': [],
-      'itemsPage': {
-        'cur': 1,
-        'num': 1,
-      },
+      'itemsHasMore': true,
       'itemSelected': null,
       'itemSelectedDetails': null,
       'itemSelectedReadability': '',
@@ -275,6 +298,18 @@ var vm = new Vue({
 
       return this.itemSelectedDetails.content || ''
     },
+    contentImages: function() {
+      if (!this.itemSelectedDetails) return []
+      return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'image')
+    },
+    contentAudios: function() {
+      if (!this.itemSelectedDetails) return []
+      return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'audio')
+    },
+    contentVideos: function() {
+      if (!this.itemSelectedDetails) return []
+      return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'video')
+    }
   },
   watch: {
     'theme': {
@@ -304,13 +339,13 @@ var vm = new Vue({
     },
     'filterSelected': function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
-      api.settings.update({filter: newVal}).then(this.refreshItems.bind(this))
+      api.settings.update({filter: newVal}).then(this.refreshItems.bind(this, false))
       this.itemSelected = null
       this.computeStats()
     },
     'feedSelected': function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
-      api.settings.update({feed: newVal}).then(this.refreshItems.bind(this))
+      api.settings.update({feed: newVal}).then(this.refreshItems.bind(this, false))
       this.itemSelected = null
       if (this.$refs.itemlist) this.$refs.itemlist.scrollTop = 0
     },
@@ -339,7 +374,7 @@ var vm = new Vue({
     }, 500),
     'itemSortNewestFirst': function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
-      api.settings.update({sort_newest_first: newVal}).then(this.refreshItems.bind(this))
+      api.settings.update({sort_newest_first: newVal}).then(vm.refreshItems.bind(this, false))
     },
     'feedListWidth': debounce(function(newVal, oldVal) {
       if (oldVal === undefined) return  // do nothing, initial setup
@@ -404,34 +439,53 @@ var vm = new Vue({
           vm.feeds = values[1]
         })
     },
-    refreshItems: function() {
+    refreshItems: function(loadMore = false) {
       if (this.feedSelected === null) {
         vm.items = []
-        vm.itemsPage = {'cur': 1, 'num': 1}
+        vm.itemsHasMore = false
         return
       }
+
       var query = this.getItemsQuery()
+      if (loadMore) {
+        query.after = vm.items[vm.items.length-1].id
+      }
+
       this.loading.items = true
       return api.items.list(query).then(function(data) {
-        vm.items = data.list
-        vm.itemsPage = data.page
+        if (loadMore) {
+          vm.items = vm.items.concat(data.list)
+        } else {
+          vm.items = data.list
+        }
+        vm.itemsHasMore = data.has_more
         vm.loading.items = false
+
+        // load more if there's some space left at the bottom of the item list.
+        vm.$nextTick(function() {
+          if (vm.itemsHasMore && !vm.loading.items && vm.itemListCloseToBottom()) {
+            vm.refreshItems(true)
+          }
+        })
       })
     },
+    itemListCloseToBottom: function() {
+      // approx. vertical space at the bottom of the list (loading el & paddings) when 1rem = 16px
+      var bottomSpace = 70
+      var scale = (parseFloat(getComputedStyle(document.documentElement).fontSize) || 16) / 16
+
+      var el = this.$refs.itemlist
+
+      if (el.scrollHeight === 0) return false  // element is invisible (responsive design)
+
+      var closeToBottom = (el.scrollHeight - el.scrollTop - el.offsetHeight) < bottomSpace * scale
+      return closeToBottom
+    },
     loadMoreItems: function(event, el) {
-      if (this.itemsPage.cur >= this.itemsPage.num) return
+      if (!this.itemsHasMore) return
       if (this.loading.items) return
-      var closeToBottom = (el.scrollHeight - el.scrollTop - el.offsetHeight) < 50
-      if (closeToBottom) {
-        this.loading.moreitems = true
-        var query = this.getItemsQuery()
-        query.page = this.itemsPage.cur + 1
-        api.items.list(query).then(function(data) {
-          vm.items = vm.items.concat(data.list)
-          vm.itemsPage = data.page
-          vm.loading.items = false
-        })
-      }
+      if (this.itemListCloseToBottom()) return this.refreshItems(true)
+      if (this.itemSelected && this.itemSelected === this.items[this.items.length - 1].id) return this.refreshItems(true)
     },
     markItemsRead: function() {
       var query = this.getItemsQuery()
@@ -439,6 +493,7 @@ var vm = new Vue({
         vm.items = []
         vm.itemsPage = {'cur': 1, 'num': 1}
         vm.itemSelected = null
+        vm.itemsHasMore = false
         vm.refreshStats()
       })
     },
@@ -498,15 +553,20 @@ var vm = new Vue({
     deleteFolder: function(folder) {
       if (confirm('Are you sure you want to delete ' + folder.title + '?')) {
         api.folders.delete(folder.id).then(function() {
-          if (vm.feedSelected === 'folder:'+folder.id) {
-            vm.items = []
-            vm.feedSelected = ''
-          }
+          vm.feedSelected = null
           vm.refreshStats()
           vm.refreshFeeds()
         })
       }
     },
+    updateFeedLink: function(feed) {
+      var newLink = prompt('Enter feed link', feed.feed_link)
+      if (newLink) {
+        api.feeds.update(feed.id, {feed_link: newLink}).then(function() {
+          feed.feed_link = newLink
+        })
+      }
+    },
     renameFeed: function(feed) {
       var newTitle = prompt('Enter new title', feed.title)
       if (newTitle) {
@@ -518,12 +578,7 @@ var vm = new Vue({
     deleteFeed: function(feed) {
       if (confirm('Are you sure you want to delete ' + feed.title + '?')) {
         api.feeds.delete(feed.id).then(function() {
-          // unselect feed to prevent reading properties of null in template
-          var isSelected = !vm.feedSelected
-            || (vm.feedSelected === 'feed:'+feed.id
-            || (feed.folder_id && vm.feedSelected === 'folder:'+feed.folder_id));
-          if (isSelected) vm.feedSelected = null
-
+          vm.feedSelected = null
           vm.refreshStats()
           vm.refreshFeeds()
         })
@@ -633,10 +688,7 @@ var vm = new Vue({
     fetchAllFeeds: function() {
       if (this.loading.feeds) return
       api.feeds.refresh().then(function() {
-        // NOTE: this is hacky
-        setTimeout(function() {
-          vm.refreshStats()
-        }, 1000)
+        vm.refreshStats()
       })
     },
     computeStats: function() {
@@ -667,6 +719,65 @@ var vm = new Vue({
       this.filteredFolderStats = statsFolders
       this.filteredTotalStats = statsTotal
     },
+    // navigation helper, navigate relative to selected item
+    navigateToItem: function(relativePosition) {
+      let vm = this
+      if (vm.itemSelected == null) {
+        // if no item is selected, select first
+        if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
+        return
+      }
+
+      var itemPosition = vm.items.findIndex(function(x) { return x.id === vm.itemSelected })
+      if (itemPosition === -1) {
+        if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
+        return
+      }
+
+      var newPosition = itemPosition + relativePosition
+      if (newPosition < 0 || newPosition >= vm.items.length) return
+
+      vm.itemSelected = vm.items[newPosition].id
+
+      vm.$nextTick(function() {
+        var scroll = document.querySelector('#item-list-scroll')
+
+        var handle = scroll.querySelector('input[type=radio]:checked')
+        var target = handle && handle.parentElement
+
+        if (target && scroll) scrollto(target, scroll)
+
+        vm.loadMoreItems()
+      })
+    },
+    // navigation helper, navigate relative to selected feed
+    navigateToFeed: function(relativePosition) {
+      let vm = this
+      var navigationList = Array.from(document.querySelectorAll('#col-feed-list input[name=feed]'))
+        .filter(function(r) { return r.offsetParent !== null && r.value !== 'folder:null' })
+        .map(function(r) { return r.value })
+
+      var currentFeedPosition = navigationList.indexOf(vm.feedSelected)
+
+      if (currentFeedPosition == -1) {
+        vm.feedSelected = ''
+        return
+      }
+
+      var newPosition = currentFeedPosition+relativePosition
+      if (newPosition < 0 || newPosition >= navigationList.length) return
+
+      vm.feedSelected = navigationList[newPosition]
+
+      vm.$nextTick(function() {
+        var scroll = document.querySelector('#feed-list-scroll')
+
+        var handle = scroll.querySelector('input[type=radio]:checked')
+        var target = handle && handle.parentElement
+
+        if (target && scroll) scrollto(target, scroll)
+      })
+    },
   }
 })
 

src/assets/javascripts/key.js

@@ -1,79 +1,4 @@
-function scrollto(target, scroll) {
-  var padding = 10
-  var targetRect = target.getBoundingClientRect()
-  var scrollRect = scroll.getBoundingClientRect()
-
-  // target
-  var relativeOffset = targetRect.y - scrollRect.y
-  var absoluteOffset = relativeOffset + scroll.scrollTop
-
-  if (padding <= relativeOffset && relativeOffset + targetRect.height <= scrollRect.height - padding) return
-
-  var newPos = scroll.scrollTop
-  if (relativeOffset < padding) {
-    newPos = absoluteOffset - padding
-  } else {
-    newPos = absoluteOffset - scrollRect.height + targetRect.height + padding
-  }
-  scroll.scrollTop = Math.round(newPos)
-}
-
 var helperFunctions = {
-  // navigation helper, navigate relative to selected item
-  navigateToItem: function(relativePosition) {
-    if (vm.itemSelected == null) {
-      // if no item is selected, select first
-      if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
-      return
-    }
-
-    var itemPosition = vm.items.findIndex(function(x) { return x.id === vm.itemSelected })
-    if (itemPosition === -1) {
-      if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
-      return
-    }
-
-    var newPosition = itemPosition + relativePosition
-    if (newPosition < 0 || newPosition >= vm.items.length) return
-
-    vm.itemSelected = vm.items[newPosition].id
-
-    vm.$nextTick(function() {
-      var scroll = document.querySelector('#item-list-scroll')
-
-      var handle = scroll.querySelector('input[type=radio]:checked')
-      var target = handle && handle.parentElement
-
-      if (target && scroll) scrollto(target, scroll)
-    })
-  },
-  // navigation helper, navigate relative to selected feed
-  navigateToFeed: function(relativePosition) {
-    var navigationList = Array.from(document.querySelectorAll('#col-feed-list input[name=feed]'))
-      .filter(function(r) { return r.offsetParent !== null && r.value !== 'folder:null' })
-      .map(function(r) { return r.value })
-
-    var currentFeedPosition = navigationList.indexOf(vm.feedSelected)
-
-    if (currentFeedPosition == -1) {
-      vm.feedSelected = ''
-      return
-    }
-
-    var newPosition = currentFeedPosition+relativePosition
-    if (newPosition < 0 || newPosition >= navigationList.length) return
-
-    vm.feedSelected = navigationList[newPosition]
-
-    vm.$nextTick(function() {
-      var scroll = document.querySelector('#feed-list-scroll')
-
-      var handle = scroll.querySelector('input[type=radio]:checked')
-      var target = handle && handle.parentElement
-
-      if (target && scroll) scrollto(target, scroll)
-    })
-  },
   scrollContent: function(direction) {
     var padding = 40
     var scroll = document.querySelector('.content')
@@ -92,7 +17,7 @@ var helperFunctions = {
 var shortcutFunctions = {
   openItemLink: function() {
     if (vm.itemSelectedDetails && vm.itemSelectedDetails.link) {
-      window.open(vm.itemSelectedDetails.link, '_blank')
+      window.open(vm.itemSelectedDetails.link, '_blank', 'noopener,noreferrer')
     }
   },
   toggleReadability: function() {
@@ -118,16 +43,16 @@ var shortcutFunctions = {
     document.getElementById("searchbar").focus()
   },
   nextItem(){
-    helperFunctions.navigateToItem(+1)
+    vm.navigateToItem(+1)
   },
   previousItem() {
-    helperFunctions.navigateToItem(-1)
+    vm.navigateToItem(-1)
   },
   nextFeed(){
-    helperFunctions.navigateToFeed(+1)
+    vm.navigateToFeed(+1)
   },
   previousFeed() {
-    helperFunctions.navigateToFeed(-1)
+    vm.navigateToFeed(-1)
   },
   scrollForward: function() {
     helperFunctions.scrollContent(+1)
@@ -165,6 +90,24 @@ var keybindings = {
   "3": shortcutFunctions.showAll,
 }
 
+var codebindings = {
+  "KeyO": shortcutFunctions.openItemLink,
+  "KeyI": shortcutFunctions.toggleReadability,
+  //"r": shortcutFunctions.toggleItemRead,
+  //"KeyR": shortcutFunctions.markAllRead,
+  "KeyS": shortcutFunctions.toggleItemStarred,
+  "Slash": shortcutFunctions.focusSearch,
+  "KeyJ": shortcutFunctions.nextItem,
+  "KeyK": shortcutFunctions.previousItem,
+  "KeyL": shortcutFunctions.nextFeed,
+  "KeyH": shortcutFunctions.previousFeed,
+  "KeyF": shortcutFunctions.scrollForward,
+  "KeyB": shortcutFunctions.scrollBackward,
+  "Digit1": shortcutFunctions.showUnread,
+  "Digit2": shortcutFunctions.showStarred,
+  "Digit3": shortcutFunctions.showAll,
+}
+
 function isTextBox(element) {
   var tagName = element.tagName.toLowerCase()
   // Input elements that aren't text
@@ -179,10 +122,10 @@ function isTextBox(element) {
 document.addEventListener('keydown',function(event) {
   // Ignore while focused on text or
   // when using modifier keys (to not clash with browser behaviour)
-  if (isTextBox(event.target) || event.metaKey || event.ctrlKey) {
+  if (isTextBox(event.target) || event.metaKey || event.ctrlKey || event.altKey) {
     return
   }
-  var keybindFunction = keybindings[event.key]
+  var keybindFunction = keybindings[event.key] || codebindings[event.code]
   if (keybindFunction) {
     event.preventDefault()
     keybindFunction()

src/assets/login.html

@@ -5,7 +5,8 @@
     <title>yarr!</title>
     <link rel="stylesheet" href="./static/stylesheets/bootstrap.min.css">
     <link rel="stylesheet" href="./static/stylesheets/app.css">
-    <link rel="icon shortcut" href="./static/graphicarts/icon.png">
+    <link rel="icon" href="./static/graphicarts/favicon.svg" type="image/svg+xml">
+    <link rel="alternate icon" href="./static/graphicarts/favicon.png" type="image/png">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
     <style>
         form {
@@ -21,7 +22,7 @@
         }
     </style>
 </head>
-<body>
+<body class="theme-{% .settings.theme_name %}">
     <form action="" method="post">
         <img src="./static/graphicarts/anchor.svg" alt="">
         {% if .error %}
@@ -30,7 +31,7 @@
         <div class="form-group">
             <label for="username">Username</label>
             <input name="username" class="form-control" id="username" autocomplete="off"
-                   value="{% if .username %}{% .username %}{% end %}" required>
+                   value="{% if .username %}{% .username %}{% end %}" required autofocus>
         </div>
         <div class="form-group">
             <label for="password">Password</label>

src/assets/stylesheets/app.css

@@ -2,8 +2,11 @@
   display: none !important;
 }
 
-body {
+html {
   font-size: 15px !important;
+}
+
+body {
   overscroll-behavior: none;
 }
 
@@ -85,6 +88,10 @@ select.form-control:not([multiple]):not([size]) {
   outline: none;
 }
 
+.table-compact {
+  color: unset !important;
+}
+
 .table-compact tr td:first-child {
   padding-left: 0;
 }
@@ -160,7 +167,9 @@ select.form-control:not([multiple]):not([size]) {
   opacity: 0;
   position: absolute;
   z-index: -1;
-  top: 0; left: 0;
+  top: 0;
+  left: 0;
+  height: 100%;
 }
 
 .selectgroup + .selectgroup {
@@ -349,6 +358,11 @@ select.form-control:not([multiple]):not([size]) {
   line-height: 1.5;
 }
 
+.content-wrapper {
+  max-width: 60rem;
+  margin: 0 auto;
+}
+
 .content img, .content video {
   max-width: 100%;
   height: auto;
@@ -360,6 +374,27 @@ select.form-control:not([multiple]):not([size]) {
   margin-bottom: 0.5rem;
 }
 
+.content .video-wrapper {
+  position: relative;
+  display: block;
+  width: 100%;
+  overflow: hidden;
+}
+
+.content .video-wrapper::before {
+  display: block;
+  padding-top: 56.25%; /* 16x9 aspect ratio */
+  content: "";
+}
+
+.content .video-wrapper iframe {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+}
+
 .content pre {
   overflow-x: auto;
   color: inherit;
@@ -395,6 +430,11 @@ select.form-control:not([multiple]):not([size]) {
   font-size: 1rem;
 }
 
+.content p {
+  margin-top: 1rem;
+  margin-bottom: 1rem;
+}
+
 /* theme: light */
 
 button.theme-light {
@@ -402,11 +442,11 @@ button.theme-light {
 }
 
 a,
-.btn-link:hover,
-.toolbar-item.active {
+.btn-link:hover {
   color: #0080d4;
 }
 
+.toolbar-item.active,
 .dropdown-item.active,
 .dropdown-item:active,
 .selectgroup input:checked + .selectgroup-label {

src/content/htmlutil/urlutils.go

@@ -2,6 +2,7 @@ package htmlutil
 
 import (
 	"net/url"
+	"strings"
 )
 
 func Any(els []string, el string, match func(string, string) bool) bool {
@@ -31,3 +32,7 @@ func URLDomain(val string) string {
 	}
 	return val
 }
+
+func IsAPossibleLink(val string) bool {
+	return strings.HasPrefix(val, "http://") || strings.HasPrefix(val, "https://")
+}

src/content/htmlutil/utils.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"regexp"
 	"strings"
+	"unicode"
 
 	"golang.org/x/net/html"
 )
@@ -61,3 +62,16 @@ func ExtractText(content string) string {
 	text = whitespaceRegex.ReplaceAllLiteralString(text, " ")
 	return text
 }
+
+func TruncateText(input string, size int) string {
+	runes := []rune(input)
+	if len(runes) <= size {
+		return input
+	}
+	for i := size - 1; i > 0; i-- {
+		if unicode.IsSpace(runes[i]) {
+			return string(runes[:i]) + " ..."
+		}
+	}
+	return input
+}

src/content/htmlutil/utils_test.go

@@ -24,3 +24,21 @@ func TestExtractText(t *testing.T) {
 		}
 	}
 }
+
+func TestTruncateText(t *testing.T) {
+	input := "Lorem ipsum — классический текст-«рыба»"
+
+	size := 30
+	want := "Lorem ipsum — классический ..."
+	have := TruncateText(input, size)
+	if want != have {
+		t.Errorf("\nsize: %d\nwant: %#v\nhave: %#v", size, want, have)
+	}
+
+	size = 1000
+	want = input
+	have = TruncateText(input, size)
+	if want != have {
+		t.Errorf("\nsize: %d\nwant: %#v\nhave: %#v", size, want, have)
+	}
+}

src/content/readability/readability.go

@@ -6,6 +6,7 @@ package readability
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -59,6 +60,9 @@ func ExtractContent(page io.Reader) (string, error) {
 			best = body
 			break
 		}
+		if best == nil {
+			return "", errors.New("failed to extract content")
+		}
 	}
 	//log.Printf("[Readability] TopCandidate: %v", topCandidate)
 

src/content/sanitizer/sanitizer.go

@@ -58,19 +58,36 @@ func Sanitize(baseURL, input string) string {
 				attrNames, htmlAttributes := sanitizeAttributes(baseURL, tagName, token.Attr)
 
 				if hasRequiredAttributes(tagName, attrNames) {
+					wrap := isVideoIframe(token)
+					if wrap {
+						buffer.WriteString(`<div class="video-wrapper">`)
+					}
+
 					if len(attrNames) > 0 {
 						buffer.WriteString("<" + tagName + " " + htmlAttributes + ">")
 					} else {
 						buffer.WriteString("<" + tagName + ">")
 					}
 
-					tagStack = append(tagStack, tagName)
+					if tagName == "iframe" {
+						// autoclose iframes
+						buffer.WriteString("</iframe>")
+						if wrap {
+							buffer.WriteString("</div>")
+						}
+					} else {
+						tagStack = append(tagStack, tagName)
+					}
 				}
 			} else if isBlockedTag(tagName) {
 				blacklistedTagDepth++
 			}
 		case html.EndTagToken:
 			tagName := token.Data
+			// iframes are autoclosed. see above
+			if tagName == "iframe" {
+				continue
+			}
 			if isValidTag(tagName) && inList(tagName, tagStack) {
 				buffer.WriteString(fmt.Sprintf("</%s>", tagName))
 			} else if isBlockedTag(tagName) {
@@ -150,7 +167,7 @@ func getExtraAttributes(tagName string) ([]string, []string) {
 	case "iframe":
 		return []string{"sandbox", "loading"}, []string{`sandbox="allow-scripts allow-same-origin allow-popups"`, `loading="lazy"`}
 	case "img":
-		return []string{"loading"}, []string{`loading="lazy"`}
+		return []string{"loading"}, []string{`loading="lazy"`, `referrerpolicy="no-referrer"`}
 	default:
 		return nil, nil
 	}
@@ -347,7 +364,6 @@ func isBlockedTag(tagName string) bool {
 }
 
 /*
-
 One or more strings separated by commas, indicating possible image sources for the user agent to use.
 
 Each string is composed of:
@@ -355,7 +371,6 @@ Each string is composed of:
 - Optionally, whitespace followed by one of:
 - A width descriptor (a positive integer directly followed by w). The width descriptor is divided by the source size given in the sizes attribute to calculate the effective pixel density.
 - A pixel density descriptor (a positive floating point number directly followed by x).
-
 */
 func sanitizeSrcsetAttr(baseURL, value string) string {
 	var sanitizedSources []string
@@ -417,3 +432,22 @@ func isValidDataAttribute(value string) bool {
 	}
 	return false
 }
+
+func isVideoIframe(token html.Token) bool {
+	videoWhitelist := map[string]bool{
+		"player.bilibili.com":      true,
+		"player.vimeo.com":         true,
+		"www.dailymotion.com":      true,
+		"www.youtube-nocookie.com": true,
+		"www.youtube.com":          true,
+	}
+	if token.Data == "iframe" {
+		for _, attr := range token.Attr {
+			if attr.Key == "src" {
+				domain := htmlutil.URLDomain(attr.Val)
+				return videoWhitelist[domain]
+			}
+		}
+	}
+	return false
+}

src/content/sanitizer/sanitizer_test.go

@@ -8,10 +8,11 @@ import "testing"
 
 func TestValidInput(t *testing.T) {
 	input := `<p>This is a <strong>text</strong> with an image: <img src="http://example.org/" alt="Test" loading="lazy">.</p>`
-	output := Sanitize("http://example.org/", input)
+	want := `<p>This is a <strong>text</strong> with an image: <img src="http://example.org/" alt="Test" loading="lazy" referrerpolicy="no-referrer">.</p>`
+	have := Sanitize("http://example.org/", input)
 
-	if input != output {
-		t.Errorf(`Wrong output: "%s" != "%s"`, input, output)
+	if have != want {
+		t.Errorf("Wrong output: \nwant: %#v\nhave: %#v", want, have)
 	}
 }
 
@@ -27,31 +28,31 @@ func TestImgWithTextDataURL(t *testing.T) {
 
 func TestImgWithDataURL(t *testing.T) {
 	input := `<img src="data:image/gif;base64,test" alt="Example">`
-	expected := `<img src="data:image/gif;base64,test" alt="Example" loading="lazy">`
-	output := Sanitize("http://example.org/", input)
+	want := `<img src="data:image/gif;base64,test" alt="Example" loading="lazy" referrerpolicy="no-referrer">`
+	have := Sanitize("http://example.org/", input)
 
-	if output != expected {
-		t.Errorf(`Wrong output: %s`, output)
+	if have != want {
+		t.Errorf("Wrong output:\nwant: %s\nhave: %s", want, have)
 	}
 }
 
 func TestImgWithSrcset(t *testing.T) {
 	input := `<img srcset="example-320w.jpg, example-480w.jpg 1.5x,   example-640w.jpg 2x, example-640w.jpg 640w" src="example-640w.jpg" alt="Example">`
-	expected := `<img srcset="http://example.org/example-320w.jpg, http://example.org/example-480w.jpg 1.5x, http://example.org/example-640w.jpg 2x, http://example.org/example-640w.jpg 640w" src="http://example.org/example-640w.jpg" alt="Example" loading="lazy">`
-	output := Sanitize("http://example.org/", input)
+	want := `<img srcset="http://example.org/example-320w.jpg, http://example.org/example-480w.jpg 1.5x, http://example.org/example-640w.jpg 2x, http://example.org/example-640w.jpg 640w" src="http://example.org/example-640w.jpg" alt="Example" loading="lazy" referrerpolicy="no-referrer">`
+	have := Sanitize("http://example.org/", input)
 
-	if output != expected {
-		t.Errorf(`Wrong output: %s`, output)
+	if have != want {
+		t.Errorf("Wrong output:\nwant: %s\nhave: %s", want, have)
 	}
 }
 
 func TestImgWithSrcsetAndDataURL(t *testing.T) {
 	input := `<img srcset="data:image/gif;base64,test" src="http://example.org/example-320w.jpg" alt="Example">`
-	expected := `<img srcset="data:image/gif;base64,test" src="http://example.org/example-320w.jpg" alt="Example" loading="lazy">`
-	output := Sanitize("http://example.org/", input)
+	want := `<img srcset="data:image/gif;base64,test" src="http://example.org/example-320w.jpg" alt="Example" loading="lazy" referrerpolicy="no-referrer">`
+	have := Sanitize("http://example.org/", input)
 
-	if output != expected {
-		t.Errorf(`Wrong output: %s`, output)
+	if have != want {
+		t.Errorf("Wrong output:\nwant: %s\nhave: %s", want, have)
 	}
 }
 
@@ -67,16 +68,16 @@ func TestSourceWithSrcsetAndMedia(t *testing.T) {
 
 func TestMediumImgWithSrcset(t *testing.T) {
 	input := `<img alt="Image for post" class="t u v ef aj" src="https://miro.medium.com/max/5460/1*aJ9JibWDqO81qMfNtqgqrw.jpeg" srcset="https://miro.medium.com/max/552/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 276w, https://miro.medium.com/max/1000/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 500w" sizes="500px" width="2730" height="3407">`
-	expected := `<img alt="Image for post" src="https://miro.medium.com/max/5460/1*aJ9JibWDqO81qMfNtqgqrw.jpeg" srcset="https://miro.medium.com/max/552/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 276w, https://miro.medium.com/max/1000/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 500w" sizes="500px" loading="lazy">`
-	output := Sanitize("http://example.org/", input)
+	want := `<img alt="Image for post" src="https://miro.medium.com/max/5460/1*aJ9JibWDqO81qMfNtqgqrw.jpeg" srcset="https://miro.medium.com/max/552/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 276w, https://miro.medium.com/max/1000/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 500w" sizes="500px" loading="lazy" referrerpolicy="no-referrer">`
+	have := Sanitize("http://example.org/", input)
 
-	if output != expected {
-		t.Errorf(`Wrong output: %s`, output)
+	if have != want {
+		t.Errorf("Wrong output:\nwant: %s\nhave: %s", want, have)
 	}
 }
 
 func TestSelfClosingTags(t *testing.T) {
-	input := `<p>This <br> is a <strong>text</strong> <br/>with an image: <img src="http://example.org/" alt="Test" loading="lazy"/>.</p>`
+	input := `<p>This <br> is a <strong>text</strong><br/>.</p>`
 	output := Sanitize("http://example.org/", input)
 
 	if input != output {
@@ -95,11 +96,11 @@ func TestTable(t *testing.T) {
 
 func TestRelativeURL(t *testing.T) {
 	input := `This <a href="/test.html">link is relative</a> and this image: <img src="../folder/image.png"/>`
-	expected := `This <a href="http://example.org/test.html" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">link is relative</a> and this image: <img src="http://example.org/folder/image.png" loading="lazy"/>`
-	output := Sanitize("http://example.org/", input)
+	want := `This <a href="http://example.org/test.html" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">link is relative</a> and this image: <img src="http://example.org/folder/image.png" loading="lazy" referrerpolicy="no-referrer"/>`
+	have := Sanitize("http://example.org/", input)
 
-	if expected != output {
-		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	if want != have {
+		t.Errorf("Wrong output:\nwant: %s\nhave: %s", want, have)
 	}
 }
 
@@ -163,6 +164,16 @@ func TestInvalidNestedTag(t *testing.T) {
 	}
 }
 
+func TestValidIFrame(t *testing.T) {
+	input := `<iframe src="http://example.org/"></iframe>`
+	want := `<iframe src="http://example.org/" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	have := Sanitize("http://example.org/", input)
+
+	if want != have {
+		t.Errorf("Wrong output:\nwant: %s\nhave: %s", want, have)
+	}
+}
+
 func TestInvalidIFrame(t *testing.T) {
 	input := `<iframe src="http://example.org/"></iframe>`
 	expected := ``
@@ -175,7 +186,7 @@ func TestInvalidIFrame(t *testing.T) {
 
 func TestIFrameWithChildElements(t *testing.T) {
 	input := `<iframe src="https://www.youtube.com/"><p>test</p></iframe>`
-	expected := `<iframe src="https://www.youtube.com/" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	expected := `<div class="video-wrapper"><iframe src="https://www.youtube.com/" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe></div>`
 	output := Sanitize("http://example.com/", input)
 
 	if expected != output {
@@ -255,7 +266,7 @@ func TestEspaceAttributes(t *testing.T) {
 
 func TestReplaceIframeURL(t *testing.T) {
 	input := `<iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0"></iframe>`
-	expected := `<iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	expected := `<div class="video-wrapper"><iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe></div>`
 	output := Sanitize("http://example.org/", input)
 
 	if expected != output {
@@ -292,3 +303,13 @@ func TestReplaceStyle(t *testing.T) {
 		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
 	}
 }
+
+func TestWrapYoutubeIFrames(t *testing.T) {
+	input := `<iframe src="https://www.youtube.com/embed/foobar"></iframe>`
+	expected := `<div class="video-wrapper"><iframe src="https://www.youtube.com/embed/foobar" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe></div>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf("Wrong output:\nwant: %v\nhave: %v", expected, output)
+	}
+}

src/content/scraper/finder.go

@@ -1,6 +1,7 @@
 package scraper
 
 import (
+	"net/url"
 	"strings"
 
 	"github.com/nkanaev/yarr/src/content/htmlutil"
@@ -35,6 +36,18 @@ func FindFeeds(body string, base string) map[string]string {
 		link := htmlutil.AbsoluteUrl(href, base)
 		if link != "" {
 			candidates[link] = name
+
+			l, err := url.Parse(link)
+			if err == nil && l.Host == "www.youtube.com" && l.Path == "/feeds/videos.xml" {
+				// https://wiki.archiveteam.org/index.php/YouTube/Technical_details#Playlists
+				channelID, found := strings.CutPrefix(l.Query().Get("channel_id"), "UC")
+				if found {
+					const url string = "https://www.youtube.com/feeds/videos.xml?playlist_id="
+					candidates[url+"UULF"+channelID] = name + " - Videos"
+					candidates[url+"UULV"+channelID] = name + " - Live Streams"
+					candidates[url+"UUSH"+channelID] = name + " - Short videos"
+				}
+			}
 		}
 	}
 

src/content/silo/url.go

@@ -0,0 +1,17 @@
+package silo
+
+import (
+	"net/url"
+	"strings"
+)
+
+func RedirectURL(link string) string {
+	if strings.HasPrefix(link, "https://www.google.com/url?") {
+		if u, err := url.Parse(link); err == nil {
+			if u2 := u.Query().Get("url"); u2 != "" {
+				return u2
+			}
+		}
+	}
+	return link
+}

src/content/silo/url_test.go

@@ -0,0 +1,24 @@
+package silo
+
+import "testing"
+
+func TestRedirectURL(t *testing.T) {
+	link := "https://www.google.com/url?rct=j&sa=t&url=https://www.cryptoglobe.com/latest/2022/08/investment-strategist-lyn-alden-explains-why-she-is-still-bullish-on-bitcoin-long-term/&ct=ga&cd=CAIyGjlkMjI1NjUyODE3ODFjMDQ6Y29tOmVuOlVT&usg=AOvVaw16C2fJtw6m8QVEbto2HCKK"
+	want := "https://www.cryptoglobe.com/latest/2022/08/investment-strategist-lyn-alden-explains-why-she-is-still-bullish-on-bitcoin-long-term/"
+	have := RedirectURL(link)
+	if have != want {
+		t.Logf("want: %s", want)
+		t.Logf("have: %s", have)
+		t.Fail()
+	}
+
+	link = "https://example.com"
+	if RedirectURL(link) != link {
+		t.Fail()
+	}
+
+	link = "https://example.com/url?url=test.com"
+	if RedirectURL(link) != link {
+		t.Fail()
+	}
+}

src/main.go

@@ -1,106 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/nkanaev/yarr/src/platform"
-	"github.com/nkanaev/yarr/src/server"
-	"github.com/nkanaev/yarr/src/storage"
-)
-
-var Version string = "0.0"
-var GitHash string = "unknown"
-
-func main() {
-	log.SetOutput(os.Stdout)
-	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
-
-	var addr, db, authfile, certfile, keyfile, basepath string
-	var ver, open bool
-	flag.StringVar(&addr, "addr", "127.0.0.1:7070", "address to run server on")
-	flag.StringVar(&authfile, "auth-file", "", "path to a file containing username:password")
-	flag.StringVar(&basepath, "base", "", "base path of the service url")
-	flag.StringVar(&certfile, "cert-file", "", "path to cert file for https")
-	flag.StringVar(&keyfile, "key-file", "", "path to key file for https")
-	flag.StringVar(&db, "db", "", "storage file path")
-	flag.BoolVar(&ver, "version", false, "print application version")
-	flag.BoolVar(&open, "open", false, "open the server in browser")
-	flag.Parse()
-
-	if ver {
-		fmt.Printf("v%s (%s)\n", Version, GitHash)
-		return
-	}
-
-	configPath, err := os.UserConfigDir()
-	if err != nil {
-		log.Fatal("Failed to get config dir: ", err)
-	}
-
-	if db == "" {
-		storagePath := filepath.Join(configPath, "yarr")
-		if err := os.MkdirAll(storagePath, 0755); err != nil {
-			log.Fatal("Failed to create app config dir: ", err)
-		}
-		db = filepath.Join(storagePath, "storage.db")
-	}
-
-	log.Printf("using db file %s", db)
-
-	var username, password string
-	if authfile != "" {
-		f, err := os.Open(authfile)
-		if err != nil {
-			log.Fatal("Failed to open auth file: ", err)
-		}
-		defer f.Close()
-		scanner := bufio.NewScanner(f)
-		for scanner.Scan() {
-			line := scanner.Text()
-			parts := strings.Split(line, ":")
-			if len(parts) != 2 {
-				log.Fatalf("Invalid auth: %v (expected `username:password`)", line)
-			}
-			username = parts[0]
-			password = parts[1]
-			break
-		}
-	}
-
-	if (certfile != "" || keyfile != "") && (certfile == "" || keyfile == "") {
-		log.Fatalf("Both cert & key files are required")
-	}
-
-	store, err := storage.New(db)
-	if err != nil {
-		log.Fatal("Failed to initialise database: ", err)
-	}
-
-	srv := server.NewServer(store, addr)
-
-	if basepath != "" {
-		srv.BasePath = "/" + strings.Trim(basepath, "/")
-	}
-
-	if certfile != "" && keyfile != "" {
-		srv.CertFile = certfile
-		srv.KeyFile = keyfile
-	}
-
-	if username != "" && password != "" {
-		srv.Username = username
-		srv.Password = password
-	}
-
-	log.Printf("starting server at %s", srv.GetAddr())
-	if open {
-		platform.Open(srv.GetAddr())
-	}
-	platform.Start(srv)
-}

src/parser/atom.go

@@ -3,7 +3,6 @@ package parser
 
 import (
 	"encoding/xml"
-	"html"
 	"io"
 	"strings"
 
@@ -47,6 +46,8 @@ type atomLinks []atomLink
 func (a *atomText) Text() string {
 	if a.Type == "html" {
 		return htmlutil.ExtractText(a.Data)
+	} else if a.Type == "xhtml" {
+		return htmlutil.ExtractText(a.XML)
 	}
 	return a.Data
 }
@@ -56,7 +57,7 @@ func (a *atomText) String() string {
 	if a.Type == "xhtml" {
 		data = a.XML
 	}
-	return html.UnescapeString(strings.TrimSpace(data))
+	return strings.TrimSpace(data)
 }
 
 func (links atomLinks) First(rel string) string {
@@ -81,15 +82,23 @@ func ParseAtom(r io.Reader) (*Feed, error) {
 		SiteURL: firstNonEmpty(srcfeed.Links.First("alternate"), srcfeed.Links.First("")),
 	}
 	for _, srcitem := range srcfeed.Entries {
-		link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""))
+		linkFromID := ""
+		guidFromID := ""
+		if htmlutil.IsAPossibleLink(srcitem.ID) {
+			linkFromID = srcitem.ID
+			guidFromID = srcitem.ID + "::" + srcitem.Updated
+		}
+
+		mediaLinks := srcitem.mediaLinks()
+
+		link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""), linkFromID)
 		dstfeed.Items = append(dstfeed.Items, Item{
-			GUID:     firstNonEmpty(srcitem.ID, link),
-			Date:     dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)),
-			URL:      link,
-			Title:    srcitem.Title.Text(),
-			Content:  firstNonEmpty(srcitem.Content.String(), srcitem.Summary.String(), srcitem.firstMediaDescription()),
-			ImageURL: srcitem.firstMediaThumbnail(),
-			AudioURL: "",
+			GUID:       firstNonEmpty(guidFromID, srcitem.ID, link),
+			Date:       dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)),
+			URL:        link,
+			Title:      srcitem.Title.Text(),
+			Content:    firstNonEmpty(srcitem.Content.String(), srcitem.Summary.String(), srcitem.firstMediaDescription()),
+			MediaLinks: mediaLinks,
 		})
 	}
 	return dstfeed, nil

src/parser/atom_test.go

@@ -40,13 +40,11 @@ func TestAtom(t *testing.T) {
 		SiteURL: "http://example.org/",
 		Items: []Item{
 			{
-				GUID:     "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a",
-				Date:     time.Unix(1071340202, 0).UTC(),
-				URL:      "http://example.org/2003/12/13/atom03.html",
-				Title:    "Atom-Powered Robots Run Amok",
-				Content:  `<div xmlns="http://www.w3.org/1999/xhtml"><p>This is the entry content.</p></div>`,
-				ImageURL: "",
-				AudioURL: "",
+				GUID:    "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a",
+				Date:    time.Unix(1071340202, 0).UTC(),
+				URL:     "http://example.org/2003/12/13/atom03.html",
+				Title:   "Atom-Powered Robots Run Amok",
+				Content: `<div xmlns="http://www.w3.org/1999/xhtml"><p>This is the entry content.</p></div>`,
 			},
 		},
 	}
@@ -94,6 +92,44 @@ func TestAtomHTMLTitle(t *testing.T) {
 	}
 }
 
+func TestAtomXHTMLTitle(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<feed xmlns="http://www.w3.org/2005/Atom">
+			<entry><title type="xhtml">say &lt;code&gt;what&lt;/code&gt;?</entry>
+		</feed>
+	`))
+	have := feed.Items[0].Title
+	want := "say what?"
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.FailNow()
+	}
+}
+
+func TestAtomXHTMLNestedTitle(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<feed xmlns="http://www.w3.org/2005/Atom">
+			<entry>
+				<title type="xhtml">
+					<div xmlns="http://www.w3.org/1999/xhtml">
+						<a href="https://example.com">Link to Example</a>
+					</div>
+				</title>
+			</entry>
+		</feed>
+	`))
+	have := feed.Items[0].Title
+	want := "Link to Example"
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.FailNow()
+	}
+}
+
 func TestAtomImageLink(t *testing.T) {
 	feed, _ := Parse(strings.NewReader(`
 		<?xml version="1.0" encoding="UTF-8"?>
@@ -103,9 +139,15 @@ func TestAtomImageLink(t *testing.T) {
 			</entry>
 		</feed>
 	`))
-	have := feed.Items[0].ImageURL
-	want := `https://example.com/image.png?width=100&height=100`
-	if want != have {
+	if len(feed.Items[0].MediaLinks) != 1 {
+		t.Fatalf("Expected 1 media link, got: %#v", feed.Items[0].MediaLinks)
+	}
+	have := feed.Items[0].MediaLinks[0]
+	want := MediaLink{
+		URL:  `https://example.com/image.png?width=100&height=100`,
+		Type: "image",
+	}
+	if !reflect.DeepEqual(want, have) {
 		t.Fatalf("item.image_url doesn't match\nwant: %#v\nhave: %#v\n", want, have)
 	}
 }
@@ -127,7 +169,68 @@ func TestAtomImageLinkDuplicated(t *testing.T) {
 	if want != have {
 		t.Fatalf("want: %#v\nhave: %#v\n", want, have)
 	}
-	if feed.Items[0].ImageURL != "" {
-		t.Fatal("item.image_url must be unset if present in the content")
+	if len(feed.Items[0].MediaLinks) != 0 {
+		t.Fatal("item media link must be excluded if present in the content")
+	}
+}
+
+func TestAtomLinkInID(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
+			<entry>
+                <title>one updated</title>
+                <id>https://example.com/posts/1</id>
+                <updated>2003-12-13T09:17:51</updated>
+			</entry>
+			<entry>
+                <title>two</title>
+                <id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
+			</entry>
+			<entry>
+                <title>one</title>
+                <id>https://example.com/posts/1</id>
+			</entry>
+		</feed>
+	`))
+	have := feed.Items
+	want := []Item{
+		Item{
+			GUID:  "https://example.com/posts/1::2003-12-13T09:17:51",
+			Date:  time.Date(2003, time.December, 13, 9, 17, 51, 0, time.UTC),
+			URL:   "https://example.com/posts/1",
+			Title: "one updated",
+		},
+		Item{
+			GUID: "urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6",
+			Date: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), URL: "",
+			Title: "two",
+		},
+		Item{
+			GUID:    "https://example.com/posts/1::",
+			Date:    time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
+			URL:     "https://example.com/posts/1",
+			Title:   "one",
+			Content: "",
+		},
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Fatalf("\nwant: %#v\nhave: %#v\n", want, have)
+	}
+}
+
+func TestAtomDoesntEscapeHTMLTags(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<feed xmlns="http://www.w3.org/2005/Atom">
+			<entry><summary type="html">&amp;lt;script&amp;gt;alert(1);&amp;lt;/script&amp;gt;</summary></entry>
+		</feed>
+	`))
+	have := feed.Items[0].Content
+	want := "&lt;script&gt;alert(1);&lt;/script&gt;"
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.FailNow()
 	}
 }

src/parser/feed.go

@@ -2,6 +2,7 @@ package parser
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -9,15 +10,27 @@ import (
 	"net/url"
 	"strings"
 	"time"
+
+	"github.com/nkanaev/yarr/src/content/htmlutil"
+	"golang.org/x/net/html/charset"
 )
 
 var UnknownFormat = errors.New("unknown feed format")
 
-type processor func(r io.Reader) (*Feed, error)
+type feedProbe struct {
+	feedType string
+	callback func(r io.Reader) (*Feed, error)
+	encoding string
+}
 
-func sniff(lookup string) (string, processor) {
+func sniff(lookup string) (out feedProbe) {
 	lookup = strings.TrimSpace(lookup)
 	lookup = strings.TrimLeft(lookup, "\x00\xEF\xBB\xBF\xFE\xFF")
+
+	if len(lookup) == 0 {
+		return
+	}
+
 	switch lookup[0] {
 	case '<':
 		decoder := xmlDecoder(strings.NewReader(lookup))
@@ -26,24 +39,42 @@ func sniff(lookup string) (string, processor) {
 			if token == nil {
 				break
 			}
+
+			// check <?xml encoding="ENCODING" ?>
+			if el, ok := token.(xml.ProcInst); ok && el.Target == "xml" {
+				out.encoding = strings.ToLower(procInst("encoding", string(el.Inst)))
+			}
+
 			if el, ok := token.(xml.StartElement); ok {
 				switch el.Name.Local {
 				case "rss":
-					return "rss", ParseRSS
+					out.feedType = "rss"
+					out.callback = ParseRSS
+					return
 				case "RDF":
-					return "rdf", ParseRDF
+					out.feedType = "rdf"
+					out.callback = ParseRDF
+					return
 				case "feed":
-					return "atom", ParseAtom
+					out.feedType = "atom"
+					out.callback = ParseAtom
+					return
 				}
 			}
 		}
 	case '{':
-		return "json", ParseJSON
+		out.feedType = "json"
+		out.callback = ParseJSON
+		return
 	}
-	return "", nil
+	return
 }
 
 func Parse(r io.Reader) (*Feed, error) {
+	return ParseWithEncoding(r, "")
+}
+
+func ParseWithEncoding(r io.Reader, fallbackEncoding string) (*Feed, error) {
 	lookup := make([]byte, 2048)
 	n, err := io.ReadFull(r, lookup)
 	switch {
@@ -56,18 +87,43 @@ func Parse(r io.Reader) (*Feed, error) {
 		r = io.MultiReader(bytes.NewReader(lookup), r)
 	}
 
-	_, callback := sniff(string(lookup))
-	if callback == nil {
+	out := sniff(string(lookup))
+	if out.feedType == "" {
 		return nil, UnknownFormat
 	}
 
-	feed, err := callback(r)
+	if out.encoding == "" && fallbackEncoding != "" {
+		r, err = charset.NewReaderLabel(fallbackEncoding, r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if (out.feedType != "json") && (out.encoding == "" || out.encoding == "utf-8") {
+		// XML decoder will not rely on custom CharsetReader (see `xmlDecoder`)
+		// to handle invalid xml characters.
+		// Assume input is already UTF-8 and do the cleanup here.
+		r = NewSafeXMLReader(r)
+	}
+
+	feed, err := out.callback(r)
 	if feed != nil {
 		feed.cleanup()
 	}
 	return feed, err
 }
 
+func ParseAndFix(r io.Reader, baseURL, fallbackEncoding string) (*Feed, error) {
+	feed, err := ParseWithEncoding(r, fallbackEncoding)
+	if err != nil {
+		return nil, err
+	}
+	feed.TranslateURLs(baseURL)
+	feed.SetMissingDatesTo(time.Now())
+	feed.SetMissingGUIDs()
+	return feed, nil
+}
+
 func (feed *Feed) cleanup() {
 	feed.Title = strings.TrimSpace(feed.Title)
 	feed.SiteURL = strings.TrimSpace(feed.SiteURL)
@@ -75,14 +131,17 @@ func (feed *Feed) cleanup() {
 	for i, item := range feed.Items {
 		feed.Items[i].GUID = strings.TrimSpace(item.GUID)
 		feed.Items[i].URL = strings.TrimSpace(item.URL)
-		feed.Items[i].Title = strings.TrimSpace(item.Title)
+		feed.Items[i].Title = strings.TrimSpace(htmlutil.ExtractText(item.Title))
 		feed.Items[i].Content = strings.TrimSpace(item.Content)
 
-		if item.ImageURL != "" && strings.Contains(item.Content, item.ImageURL) {
-			feed.Items[i].ImageURL = ""
-		}
-		if item.AudioURL != "" && strings.Contains(item.Content, item.AudioURL) {
-			feed.Items[i].AudioURL = ""
+		if len(feed.Items[i].MediaLinks) > 0 {
+			mediaLinks := make([]MediaLink, 0)
+			for _, link := range item.MediaLinks {
+				if !strings.Contains(item.Content, link.URL) {
+					mediaLinks = append(mediaLinks, link)
+				}
+			}
+			feed.Items[i].MediaLinks = mediaLinks
 		}
 	}
 }
@@ -114,3 +173,12 @@ func (feed *Feed) TranslateURLs(base string) error {
 	}
 	return nil
 }
+
+func (feed *Feed) SetMissingGUIDs() {
+	for i, item := range feed.Items {
+		if item.GUID == "" {
+			id := strings.Join([]string{item.Title, item.Date.Format(time.RFC3339), item.URL}, ";;")
+			feed.Items[i].GUID = fmt.Sprintf("%x", sha256.Sum256([]byte(id)))
+		}
+	}
+}

src/parser/feed_test.go

@@ -7,38 +7,40 @@ import (
 )
 
 func TestSniff(t *testing.T) {
-	testcases := [][2]string{
+	testcases := []struct {
+		input string
+		want  feedProbe
+	}{
 		{
 			`<?xml version="1.0"?><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"></rdf:RDF>`,
-			"rdf",
+			feedProbe{feedType: "rdf", callback: ParseRDF},
 		},
 		{
 			`<?xml version="1.0" encoding="ISO-8859-1"?><rss version="2.0"><channel></channel></rss>`,
-			"rss",
+			feedProbe{feedType: "rss", callback: ParseRSS, encoding: "iso-8859-1"},
 		},
 		{
 			`<?xml version="1.0"?><rss version="2.0"><channel></channel></rss>`,
-			"rss",
+			feedProbe{feedType: "rss", callback: ParseRSS},
 		},
 		{
 			`<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>`,
-			"atom",
+			feedProbe{feedType: "atom", callback: ParseAtom, encoding: "utf-8"},
 		},
 		{
 			`{}`,
-			"json",
+			feedProbe{feedType: "json", callback: ParseJSON},
 		},
 		{
 			`<!DOCTYPE html><html><head><title></title></head><body></body></html>`,
-			"",
+			feedProbe{},
 		},
 	}
 	for _, testcase := range testcases {
-		have, _ := sniff(testcase[0])
-		want := testcase[1]
-		if want != have {
-			t.Log(testcase[0])
-			t.Errorf("Invalid format: want=%#v have=%#v", want, have)
+		want := testcase.want
+		have := sniff(testcase.input)
+		if want.encoding != have.encoding || want.feedType != have.feedType {
+			t.Errorf("Invalid output\n---\n%s\n---\n\nwant=%#v\nhave=%#v", testcase.input, want, have)
 		}
 	}
 }
@@ -107,3 +109,73 @@ func TestParseFeedWithBOM(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+func TestParseCleanIllegalCharsInUTF8(t *testing.T) {
+	data := `
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+					<title>` + "\a" + `title</title>
+				</item>
+			</channel>
+		</rss>
+	`
+	feed, err := Parse(strings.NewReader(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(feed.Items) != 1 || feed.Items[0].Title != "title" {
+		t.Fatalf("invalid feed, got: %v", feed)
+	}
+}
+
+func TestParseCleanIllegalCharsInNonUTF8(t *testing.T) {
+	// echo привет | iconv -f utf8 -t cp1251 | hexdump -C
+	data := `
+		<?xml version="1.0" encoding="windows-1251"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+					<title>` + "\a \xef\xf0\xe8\xe2\xe5\xf2\x0a \a" + `</title>
+				</item>
+			</channel>
+		</rss>
+	`
+	feed, err := Parse(strings.NewReader(data))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(feed.Items) != 1 || feed.Items[0].Title != "привет" {
+		t.Fatalf("invalid feed, got: %v", feed)
+	}
+}
+
+func TestParseMissingGUID(t *testing.T) {
+	data := `
+		<?xml version="1.0" encoding="windows-1251"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+					<title>foo</title>
+				</item>
+				<item>
+					<title>bar</title>
+				</item>
+			</channel>
+		</rss>
+	`
+	feed, err := ParseAndFix(strings.NewReader(data), "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(feed.Items) != 2 {
+		t.Fatalf("expected 2 items, got %d", len(feed.Items))
+	}
+	if feed.Items[0].GUID == "" || feed.Items[1].GUID == "" {
+		t.Fatalf("item GUIDs are missing, got %#v", feed.Items)
+	}
+	if feed.Items[0].GUID == feed.Items[1].GUID {
+		t.Fatalf("item GUIDs are not unique, got %#v", feed.Items)
+	}
+}

src/parser/media.go

@@ -1,5 +1,9 @@
 package parser
 
+import (
+	"strings"
+)
+
 type media struct {
 	MediaGroups       []mediaGroup       `xml:"http://search.yahoo.com/mrss/ group"`
 	MediaContents     []mediaContent     `xml:"http://search.yahoo.com/mrss/ content"`
@@ -8,12 +12,17 @@ type media struct {
 }
 
 type mediaGroup struct {
+	MediaContent      []mediaContent     `xml:"http://search.yahoo.com/mrss/ content"`
 	MediaThumbnails   []mediaThumbnail   `xml:"http://search.yahoo.com/mrss/ thumbnail"`
 	MediaDescriptions []mediaDescription `xml:"http://search.yahoo.com/mrss/ description"`
 }
 
 type mediaContent struct {
-	MediaThumbnails []mediaThumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"`
+	MediaThumbnails  []mediaThumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"`
+	MediaType        string           `xml:"type,attr"`
+	MediaMedium      string           `xml:"medium,attr"`
+	MediaURL         string           `xml:"url,attr"`
+	MediaDescription mediaDescription `xml:"http://search.yahoo.com/mrss/ description"`
 }
 
 type mediaThumbnail struct {
@@ -21,8 +30,8 @@ type mediaThumbnail struct {
 }
 
 type mediaDescription struct {
-	Type        string `xml:"type,attr"`
-	Description string `xml:",chardata"`
+	Type string `xml:"type,attr"`
+	Text string `xml:",chardata"`
 }
 
 func (m *media) firstMediaThumbnail() string {
@@ -44,12 +53,59 @@ func (m *media) firstMediaThumbnail() string {
 
 func (m *media) firstMediaDescription() string {
 	for _, d := range m.MediaDescriptions {
-		return plain2html(d.Description)
+		return plain2html(d.Text)
 	}
 	for _, g := range m.MediaGroups {
 		for _, d := range g.MediaDescriptions {
-			return plain2html(d.Description)
+			return plain2html(d.Text)
 		}
 	}
 	return ""
 }
+
+func (m *media) mediaLinks() []MediaLink {
+	links := make([]MediaLink, 0)
+	for _, thumbnail := range m.MediaThumbnails {
+		links = append(links, MediaLink{URL: thumbnail.URL, Type: "image"})
+	}
+	for _, group := range m.MediaGroups {
+		for _, thumbnail := range group.MediaThumbnails {
+			links = append(links, MediaLink{
+				URL:  thumbnail.URL,
+				Type: "image",
+			})
+		}
+	}
+	for _, content := range m.MediaContents {
+		if content.MediaURL != "" {
+			url := content.MediaURL
+			description := content.MediaDescription.Text
+			if strings.HasPrefix(content.MediaType, "image/") {
+				links = append(links, MediaLink{URL: url, Type: "image", Description: description})
+			} else if strings.HasPrefix(content.MediaType, "audio/") {
+				links = append(links, MediaLink{URL: url, Type: "audio", Description: description})
+			} else if strings.HasPrefix(content.MediaType, "video/") {
+				links = append(links, MediaLink{URL: url, Type: "video", Description: description})
+			} else if content.MediaMedium == "image" || content.MediaMedium == "audio" || content.MediaMedium == "video" {
+				links = append(links, MediaLink{URL: url, Type: content.MediaMedium, Description: description})
+			} else {
+				if len(content.MediaThumbnails) > 0 {
+					links = append(links, MediaLink{
+						URL:  content.MediaThumbnails[0].URL,
+						Type: "image",
+					})
+				}
+			}
+		}
+		for _, thumbnail := range content.MediaThumbnails {
+			links = append(links, MediaLink{
+				URL:  thumbnail.URL,
+				Type: "image",
+			})
+		}
+	}
+	if len(links) == 0 {
+		return nil
+	}
+	return links
+}

src/parser/models.go

@@ -14,7 +14,12 @@ type Item struct {
 	URL   string
 	Title string
 
-	Content  string
-	ImageURL string
-	AudioURL string
+	Content    string
+	MediaLinks []MediaLink
+}
+
+type MediaLink struct {
+	URL         string
+	Type        string
+	Description string
 }

src/parser/rss.go

@@ -20,9 +20,9 @@ type rssFeed struct {
 }
 
 type rssItem struct {
-	GUID        string         `xml:"guid"`
+	GUID        rssGuid        `xml:"guid"`
 	Title       string         `xml:"title"`
-	Link        string         `xml:"link"`
+	Link        string         `xml:"rss link"`
 	Description string         `xml:"rss description"`
 	PubDate     string         `xml:"pubDate"`
 	Enclosures  []rssEnclosure `xml:"enclosure"`
@@ -36,6 +36,11 @@ type rssItem struct {
 	media
 }
 
+type rssGuid struct {
+	GUID        string `xml:",chardata"`
+	IsPermaLink string `xml:"isPermaLink,attr"`
+}
+
 type rssLink struct {
 	XMLName xml.Name
 	Data    string `xml:",chardata"`
@@ -69,26 +74,30 @@ func ParseRSS(r io.Reader) (*Feed, error) {
 		SiteURL: srcfeed.Link,
 	}
 	for _, srcitem := range srcfeed.Items {
-		podcastURL := ""
+		mediaLinks := srcitem.mediaLinks()
 		for _, e := range srcitem.Enclosures {
-			if e.Type == "audio/mpeg" || e.Type == "audio/x-m4a" {
-				podcastURL = e.URL
-
+			if strings.HasPrefix(e.Type, "audio/") {
+				podcastURL := e.URL
 				if srcitem.OrigEnclosureLink != "" && strings.Contains(podcastURL, path.Base(srcitem.OrigEnclosureLink)) {
 					podcastURL = srcitem.OrigEnclosureLink
 				}
+				mediaLinks = append(mediaLinks, MediaLink{URL: podcastURL, Type: "audio"})
 				break
 			}
 		}
 
+		permalink := ""
+		if srcitem.GUID.IsPermaLink == "true" {
+			permalink = srcitem.GUID.GUID
+		}
+
 		dstfeed.Items = append(dstfeed.Items, Item{
-			GUID:     firstNonEmpty(srcitem.GUID, srcitem.Link),
-			Date:     dateParse(firstNonEmpty(srcitem.DublinCoreDate, srcitem.PubDate)),
-			URL:      firstNonEmpty(srcitem.OrigLink, srcitem.Link),
-			Title:    srcitem.Title,
-			Content:  firstNonEmpty(srcitem.ContentEncoded, srcitem.Description),
-			AudioURL: podcastURL,
-			ImageURL: srcitem.firstMediaThumbnail(),
+			GUID:       firstNonEmpty(srcitem.GUID.GUID, srcitem.Link),
+			Date:       dateParse(firstNonEmpty(srcitem.DublinCoreDate, srcitem.PubDate)),
+			URL:        firstNonEmpty(srcitem.OrigLink, srcitem.Link, permalink),
+			Title:      srcitem.Title,
+			Content:    firstNonEmpty(srcitem.ContentEncoded, srcitem.Description, srcitem.firstMediaDescription()),
+			MediaLinks: mediaLinks,
 		})
 	}
 	return dstfeed, nil

src/parser/rss_test.go

@@ -75,9 +75,15 @@ func TestRSSMediaContentThumbnail(t *testing.T) {
 			</channel>
 		</rss>
 	`))
-	have := feed.Items[0].ImageURL
-	want := "https://i.vimeocdn.com/video/1092705247_960.jpg"
-	if have != want {
+	if len(feed.Items[0].MediaLinks) != 1 {
+		t.Fatalf("Expected 1 media link, got %#v", feed.Items[0].MediaLinks)
+	}
+	have := feed.Items[0].MediaLinks[0]
+	want := MediaLink{
+		URL:  "https://i.vimeocdn.com/video/1092705247_960.jpg",
+		Type: "image",
+	}
+	if !reflect.DeepEqual(want, have) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.FailNow()
@@ -127,9 +133,41 @@ func TestRSSPodcast(t *testing.T) {
 			</channel>
 		</rss>
 	`))
-	have := feed.Items[0].AudioURL
-	want := "http://example.com/audio.ext"
-	if want != have {
+	if len(feed.Items[0].MediaLinks) != 1 {
+		t.Fatal("Invalid media links")
+	}
+	have := feed.Items[0].MediaLinks[0]
+	want := MediaLink{
+		URL:  "http://example.com/audio.ext",
+		Type: "audio",
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.FailNow()
+	}
+}
+
+func TestRSSOpusPodcast(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0">
+			<channel>
+				<item>
+					<enclosure length="100500" type="audio/opus" url="http://example.com/audio.ext"/>
+				</item>
+			</channel>
+		</rss>
+	`))
+	if len(feed.Items[0].MediaLinks) != 1 {
+		t.Fatal("Invalid media links")
+	}
+	have := feed.Items[0].MediaLinks[0]
+	want := MediaLink{
+		URL:  "http://example.com/audio.ext",
+		Type: "audio",
+	}
+	if !reflect.DeepEqual(want, have) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.FailNow()
@@ -156,7 +194,95 @@ func TestRSSPodcastDuplicated(t *testing.T) {
 	if want != have {
 		t.Fatalf("content doesn't match\nwant: %#v\nhave: %#v\n", want, have)
 	}
-	if feed.Items[0].AudioURL != "" {
-		t.Fatal("item.audio_url must be unset if present in the content")
+
+	if len(feed.Items[0].MediaLinks) != 0 {
+		t.Fatal("item media must be excluded if present in the content")
+	}
+}
+
+func TestRSSTitleHTMLTags(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+					<title>&lt;p&gt;title in p&lt;/p&gt;</title>
+				</item>
+				<item>
+					<title>very &lt;strong&gt;strong&lt;/strong&gt; title</title>
+				</item>
+			</channel>
+		</rss>
+	`))
+	have := []string{feed.Items[0].Title, feed.Items[1].Title}
+	want := []string{"title in p", "very strong title"}
+	for i := 0; i < len(want); i++ {
+		if want[i] != have[i] {
+			t.Errorf("title doesn't match\nwant: %#v\nhave: %#v\n", want[i], have[i])
+		}
+	}
+}
+
+func TestRSSIsPermalink(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
+			<channel>
+				<item>
+                    <guid isPermaLink="true">http://example.com/posts/1</guid>
+				</item>
+			</channel>
+		</rss>
+	`))
+	have := feed.Items
+	want := []Item{
+		{
+			GUID: "http://example.com/posts/1",
+			URL:  "http://example.com/posts/1",
+		},
+	}
+	for i := 0; i < len(want); i++ {
+		if !reflect.DeepEqual(want, have) {
+			t.Errorf("Failed to handle isPermalink\nwant: %#v\nhave: %#v\n", want[i], have[i])
+		}
+	}
+}
+
+func TestRSSMultipleMedia(t *testing.T) {
+	feed, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="UTF-8"?>
+		<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
+			<channel>
+				<item>
+                    <guid isPermaLink="true">http://example.com/posts/1</guid>
+					<media:content url="https://example.com/path/to/image1.png" type="image/png" fileSize="1000" medium="image">
+						<media:description type="plain">description 1</media:description>
+					</media:content>
+					<media:content url="https://example.com/path/to/image2.png" type="image/png" fileSize="2000" medium="image">
+						<media:description type="plain">description 2</media:description>
+					</media:content>
+					<media:content url="https://example.com/path/to/video1.mp4" type="video/mp4" fileSize="2000" medium="image">
+						<media:description type="plain">video description</media:description>
+					</media:content>
+				</item>
+			</channel>
+		</rss>
+	`))
+	have := feed.Items
+	want := []Item{
+		{
+			GUID: "http://example.com/posts/1",
+			URL:  "http://example.com/posts/1",
+			MediaLinks: []MediaLink{
+				{URL:"https://example.com/path/to/image1.png", Type:"image", Description:"description 1"},
+				{URL:"https://example.com/path/to/image2.png", Type:"image", Description:"description 2"},
+				{URL:"https://example.com/path/to/video1.mp4", Type:"video", Description:"video description"},
+			},
+		},
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid rss")
 	}
 }

src/parser/util.go

@@ -1,6 +1,8 @@
 package parser
 
 import (
+	"bufio"
+	"bytes"
 	"encoding/xml"
 	"io"
 	"regexp"
@@ -30,6 +32,81 @@ func plain2html(text string) string {
 func xmlDecoder(r io.Reader) *xml.Decoder {
 	decoder := xml.NewDecoder(r)
 	decoder.Strict = false
-	decoder.CharsetReader = charset.NewReaderLabel
+	decoder.CharsetReader = func(cs string, input io.Reader) (io.Reader, error) {
+		r, err := charset.NewReaderLabel(cs, input)
+		if err == nil {
+			r = NewSafeXMLReader(r)
+		}
+		return r, err
+	}
 	return decoder
 }
+
+type safexmlreader struct {
+	reader *bufio.Reader
+	buffer *bytes.Buffer
+}
+
+func NewSafeXMLReader(r io.Reader) io.Reader {
+	return &safexmlreader{
+		reader: bufio.NewReader(r),
+		buffer: bytes.NewBuffer(make([]byte, 0, 4096)),
+	}
+}
+
+func (xr *safexmlreader) Read(p []byte) (int, error) {
+	for xr.buffer.Len() < cap(p) {
+		r, _, err := xr.reader.ReadRune()
+		if err == io.EOF {
+			if xr.buffer.Len() == 0 {
+				return 0, io.EOF
+			}
+			break
+		}
+		if err != nil {
+			return 0, err
+		}
+		if isInCharacterRange(r) {
+			xr.buffer.WriteRune(r)
+		}
+	}
+	return xr.buffer.Read(p)
+}
+
+// NOTE: copied from "encoding/xml" package
+// Decide whether the given rune is in the XML Character Range, per
+// the Char production of https://www.xml.com/axml/testaxml.htm,
+// Section 2.2 Characters.
+func isInCharacterRange(r rune) (inrange bool) {
+	return r == 0x09 ||
+		r == 0x0A ||
+		r == 0x0D ||
+		r >= 0x20 && r <= 0xD7FF ||
+		r >= 0xE000 && r <= 0xFFFD ||
+		r >= 0x10000 && r <= 0x10FFFF
+}
+
+// NOTE: copied from "encoding/xml" package
+// procInst parses the `param="..."` or `param='...'`
+// value out of the provided string, returning "" if not found.
+func procInst(param, s string) string {
+	// TODO: this parsing is somewhat lame and not exact.
+	// It works for all actual cases, though.
+	param = param + "="
+	idx := strings.Index(s, param)
+	if idx == -1 {
+		return ""
+	}
+	v := s[idx+len(param):]
+	if v == "" {
+		return ""
+	}
+	if v[0] != '\'' && v[0] != '"' {
+		return ""
+	}
+	idx = strings.IndexRune(v[1:], rune(v[0]))
+	if idx == -1 {
+		return ""
+	}
+	return v[1 : idx+1]
+}

src/parser/util_test.go

@@ -0,0 +1,88 @@
+package parser
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+)
+
+func TestSafeXMLReader(t *testing.T) {
+	var f io.Reader
+	want := []byte("привет мир")
+	f = bytes.NewReader(want)
+	f = NewSafeXMLReader(f)
+
+	have, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Fatalf("invalid output\nwant: %v\nhave: %v", want, have)
+	}
+}
+
+func TestSafeXMLReaderRemoveUnwantedRunes(t *testing.T) {
+	var f io.Reader
+	input := []byte("\aпривет \x0cмир\ufffe\uffff")
+	want := []byte("привет мир")
+	f = bytes.NewReader(input)
+	f = NewSafeXMLReader(f)
+
+	have, err := io.ReadAll(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Fatalf("invalid output\nwant: %v\nhave: %v", want, have)
+	}
+}
+
+func TestSafeXMLReaderPartial1(t *testing.T) {
+	var f io.Reader
+	input := []byte("\aпривет \x0cмир\ufffe\uffff")
+	want := []byte("привет мир")
+	f = bytes.NewReader(input)
+	f = NewSafeXMLReader(f)
+
+	buf := make([]byte, 1)
+	for i := 0; i < len(want); i++ {
+		n, err := f.Read(buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if n != 1 {
+			t.Fatalf("expected 1 byte, got %d", n)
+		}
+		if buf[0] != want[i] {
+			t.Fatalf("invalid char at pos %d\nwant: %v\nhave: %v", i, want[i], buf[0])
+		}
+	}
+	if x, err := f.Read(buf); err != io.EOF {
+		t.Fatalf("expected EOF, %v, %v %v", buf, x, err)
+	}
+}
+
+func TestSafeXMLReaderPartial2(t *testing.T) {
+	var f io.Reader
+	input := []byte("привет\a\a\a\a\a")
+	f = bytes.NewReader(input)
+	f = NewSafeXMLReader(f)
+
+	buf := make([]byte, 12)
+	n, err := f.Read(buf)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if n != 12 {
+		t.Fatalf("expected 12 bytes")
+	}
+
+	n, err = f.Read(buf)
+	if n != 0 {
+		t.Fatalf("expected 0")
+	}
+	if err != io.EOF {
+		t.Fatalf("expected EOF, got %v", err)
+	}
+}

src/platform/fixconsole_default.go

@@ -0,0 +1,14 @@
+//go:build !windows
+
+package platform
+
+// On non-windows platforms, we don't need to do anything. The console
+// starts off attached already, if it exists.
+
+func AttachConsole() error {
+	return nil
+}
+
+func FixConsoleIfNeeded() error {
+	return nil
+}

src/platform/fixconsole_windows.go

@@ -0,0 +1,136 @@
+//go:build windows
+
+package platform
+
+import (
+	"fmt"
+	"golang.org/x/sys/windows"
+	"os"
+	"syscall"
+)
+
+func AttachConsole() error {
+	const ATTACH_PARENT_PROCESS = ^uintptr(0)
+	proc := syscall.MustLoadDLL("kernel32.dll").MustFindProc("AttachConsole")
+	r1, _, err := proc.Call(ATTACH_PARENT_PROCESS)
+	if r1 == 0 {
+		errno, ok := err.(syscall.Errno)
+		if ok && errno == windows.ERROR_INVALID_HANDLE {
+			// console handle doesn't exist; not a real
+			// error, but the console handle will be
+			// invalid.
+			return nil
+		}
+		return err
+	} else {
+		return nil
+	}
+}
+
+var oldStdin, oldStdout, oldStderr *os.File
+
+// Windows console output is a mess.
+//
+// If you compile as "-H windows", then if you launch your program without
+// a console, Windows forcibly creates one to use as your stdin/stdout, which
+// is silly for a GUI app, so we can't do that.
+//
+// If you compile as "-H windowsgui", then it doesn't create a console for
+// your app... but also doesn't provide a working stdin/stdout/stderr even if
+// you *did* launch from the console.  However, you can use AttachConsole()
+// to get a handle to your parent process's console, if any, and then
+// os.NewFile() to turn that handle into a fd usable as stdout/stderr.
+//
+// However, then you have the problem that if you redirect stdout or stderr
+// from the shell, you end up ignoring the redirection by forcing it to the
+// console.
+//
+// To fix *that*, we have to detect whether there was a pre-existing stdout
+// or not. We can check GetStdHandle(), which returns 0 for "should be
+// console" and nonzero for "already pointing at a file."
+//
+// Be careful though!  As soon as you run AttachConsole(), it resets *all*
+// the GetStdHandle() handles to point them at the console instead, thus
+// throwing away the original file redirects.  So we have to GetStdHandle()
+// *before* AttachConsole().
+//
+// For some reason, powershell redirections provide a valid file handle, but
+// writing to that handle doesn't write to the file.  I haven't found a way
+// to work around that.  (Windows 10.0.17763.379)
+//
+// Net result is as follows.
+// Before:
+//
+//	SHELL            NON-REDIRECTED     REDIRECTED
+//	explorer.exe     no console         n/a
+//	cmd.exe          broken             works
+//	powershell       broken             broken
+//	WSL bash         broken             works
+//
+// After
+//
+//	SHELL            NON-REDIRECTED     REDIRECTED
+//	explorer.exe     no console         n/a
+//	cmd.exe          works              works
+//	powershell       works              broken
+//	WSL bash         works              works
+//
+// We don't seem to make anything worse, at least.
+func FixConsoleIfNeeded() error {
+	// Retain the original console objects, to prevent Go from automatically
+	// closing their file descriptors when they get garbage collected.
+	// You never want to close file descriptors 0, 1, and 2.
+	oldStdin, oldStdout, oldStderr = os.Stdin, os.Stdout, os.Stderr
+
+	stdin, _ := syscall.GetStdHandle(syscall.STD_INPUT_HANDLE)
+	stdout, _ := syscall.GetStdHandle(syscall.STD_OUTPUT_HANDLE)
+	stderr, _ := syscall.GetStdHandle(syscall.STD_ERROR_HANDLE)
+
+	var invalid syscall.Handle
+	con := invalid
+
+	if stdin == invalid || stdout == invalid || stderr == invalid {
+		err := AttachConsole()
+		if err != nil {
+			return fmt.Errorf("attachconsole: %v", err)
+		}
+
+		if stdin == invalid {
+			stdin, _ = syscall.GetStdHandle(syscall.STD_INPUT_HANDLE)
+		}
+		if stdout == invalid {
+			stdout, _ = syscall.GetStdHandle(syscall.STD_OUTPUT_HANDLE)
+			con = stdout
+		}
+		if stderr == invalid {
+			stderr, _ = syscall.GetStdHandle(syscall.STD_ERROR_HANDLE)
+			con = stderr
+		}
+	}
+
+	if con != invalid {
+		// Make sure the console is configured to convert
+		// \n to \r\n, like Go programs expect.
+		h := windows.Handle(con)
+		var st uint32
+		err := windows.GetConsoleMode(h, &st)
+		if err != nil {
+			return fmt.Errorf("GetConsoleMode: %v", err)
+		}
+		err = windows.SetConsoleMode(h, st&^windows.DISABLE_NEWLINE_AUTO_RETURN)
+		if err != nil {
+			return fmt.Errorf("SetConsoleMode: %v", err)
+		}
+	}
+
+	if stdin != invalid {
+		os.Stdin = os.NewFile(uintptr(stdin), "stdin")
+	}
+	if stdout != invalid {
+		os.Stdout = os.NewFile(uintptr(stdout), "stdout")
+	}
+	if stderr != invalid {
+		os.Stderr = os.NewFile(uintptr(stderr), "stderr")
+	}
+	return nil
+}

src/platform/gui.go

@@ -1,4 +1,4 @@
-// +build macos windows
+//go:build (darwin || windows) && gui
 
 package platform
 
@@ -10,6 +10,7 @@ import (
 func Start(s *server.Server) {
 	systrayOnReady := func() {
 		systray.SetIcon(Icon)
+		systray.SetTooltip("yarr")
 
 		menuOpen := systray.AddMenuItem("Open", "")
 		systray.AddSeparator()

src/platform/guiless.go

@@ -1,4 +1,4 @@
-// +build !windows,!macos
+//go:build !gui
 
 package platform
 

src/platform/icon_mac.go

@@ -1,4 +1,4 @@
-// +build macos
+//go:build darwin && gui
 
 package platform
 

src/platform/icon_win.go

@@ -1,4 +1,4 @@
-// +build windows
+//go:build windows && gui
 
 package platform
 

src/platform/open_etc.go

@@ -1,4 +1,4 @@
-// +build !windows,!darwin
+//go:build linux
 
 package platform
 

src/platform/open_mac.go

@@ -1,4 +1,4 @@
-// +build darwin
+//go:build darwin
 
 package platform
 

src/platform/open_win.go

@@ -1,4 +1,4 @@
-// +build windows
+//go:build windows
 
 package platform
 

src/server/auth/middleware.go

@@ -6,13 +6,15 @@ import (
 
 	"github.com/nkanaev/yarr/src/assets"
 	"github.com/nkanaev/yarr/src/server/router"
+	"github.com/nkanaev/yarr/src/storage"
 )
 
 type Middleware struct {
 	Username string
 	Password string
 	BasePath string
-	Public   string
+	Public   []string
+	DB       *storage.Storage
 }
 
 func unsafeMethod(method string) bool {
@@ -20,9 +22,11 @@ func unsafeMethod(method string) bool {
 }
 
 func (m *Middleware) Handler(c *router.Context) {
-	if strings.HasPrefix(c.Req.URL.Path, m.BasePath+m.Public) {
-		c.Next()
-		return
+	for _, path := range m.Public {
+		if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) {
+			c.Next()
+			return
+		}
 	}
 	if IsAuthenticated(c.Req, m.Username, m.Password) {
 		c.Next()
@@ -44,12 +48,15 @@ func (m *Middleware) Handler(c *router.Context) {
 			c.Redirect(rootUrl)
 			return
 		} else {
-			c.HTML(http.StatusOK, assets.Template("login.html"), map[string]string{
+			c.HTML(http.StatusOK, assets.Template("login.html"), map[string]interface{}{
 				"username": username,
 				"error":    "Invalid username/password",
+				"settings": m.DB.GetSettings(),
 			})
 			return
 		}
 	}
-	c.HTML(http.StatusOK, assets.Template("login.html"), nil)
+	c.HTML(http.StatusOK, assets.Template("login.html"), map[string]interface{}{
+		"settings": m.DB.GetSettings(),
+	})
 }

src/server/fever.go

@@ -0,0 +1,393 @@
+package server
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/nkanaev/yarr/src/server/auth"
+	"github.com/nkanaev/yarr/src/server/router"
+	"github.com/nkanaev/yarr/src/storage"
+)
+
+type FeverGroup struct {
+	ID    int64  `json:"id"`
+	Title string `json:"title"`
+}
+
+type FeverFeedsGroup struct {
+	GroupID int64  `json:"group_id"`
+	FeedIDs string `json:"feed_ids"`
+}
+
+type FeverFeed struct {
+	ID          int64  `json:"id"`
+	FaviconID   int64  `json:"favicon_id"`
+	Title       string `json:"title"`
+	Url         string `json:"url"`
+	SiteUrl     string `json:"site_url"`
+	IsSpark     int    `json:"is_spark"`
+	LastUpdated int64  `json:"last_updated_on_time"`
+}
+
+type FeverItem struct {
+	ID        int64  `json:"id"`
+	FeedID    int64  `json:"feed_id"`
+	Title     string `json:"title"`
+	Author    string `json:"author"`
+	HTML      string `json:"html"`
+	Url       string `json:"url"`
+	IsSaved   int    `json:"is_saved"`
+	IsRead    int    `json:"is_read"`
+	CreatedAt int64  `json:"created_on_time"`
+}
+
+type FeverFavicon struct {
+	ID   int64  `json:"id"`
+	Data string `json:"data"`
+}
+
+func writeFeverJSON(c *router.Context, data map[string]interface{}, lastRefreshed int64) {
+	data["api_version"] = 3
+	data["auth"] = 1
+	data["last_refreshed_on_time"] = lastRefreshed
+	c.JSON(http.StatusOK, data)
+}
+
+func getLastRefreshedOnTime(httpStates map[int64]storage.HTTPState) int64 {
+	if len(httpStates) == 0 {
+		return 0
+	}
+
+	var lastRefreshed int64
+	for _, state := range httpStates {
+		if state.LastRefreshed.Unix() > lastRefreshed {
+			lastRefreshed = state.LastRefreshed.Unix()
+		}
+	}
+	return lastRefreshed
+}
+
+func (s *Server) feverAuth(c *router.Context) bool {
+	if s.Username != "" && s.Password != "" {
+		apiKey := c.Req.FormValue("api_key")
+		apiKey = strings.ToLower(apiKey)
+		md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password)))
+		hexMD5HashValue := fmt.Sprintf("%x", md5HashValue[:])
+		if !auth.StringsEqual(apiKey, hexMD5HashValue) {
+			return false
+		}
+	}
+	return true
+}
+
+func formHasValue(values url.Values, value string) bool {
+	if _, ok := values[value]; ok {
+		return true
+	}
+	return false
+}
+
+func (s *Server) handleFever(c *router.Context) {
+	c.Req.ParseForm()
+	if !s.feverAuth(c) {
+		c.JSON(http.StatusOK, map[string]interface{}{
+			"api_version":            3,
+			"auth":                   0,
+			"last_refreshed_on_time": 0,
+		})
+		return
+	}
+
+	switch {
+	case formHasValue(c.Req.Form, "groups"):
+		s.feverGroupsHandler(c)
+	case formHasValue(c.Req.Form, "feeds"):
+		s.feverFeedsHandler(c)
+	case formHasValue(c.Req.Form, "unread_item_ids"):
+		s.feverUnreadItemIDsHandler(c)
+	case formHasValue(c.Req.Form, "saved_item_ids"):
+		s.feverSavedItemIDsHandler(c)
+	case formHasValue(c.Req.Form, "favicons"):
+		s.feverFaviconsHandler(c)
+	case formHasValue(c.Req.Form, "items"):
+		s.feverItemsHandler(c)
+	case formHasValue(c.Req.Form, "links"):
+		s.feverLinksHandler(c)
+	case formHasValue(c.Req.Form, "mark"):
+		s.feverMarkHandler(c)
+	default:
+		c.JSON(http.StatusOK, map[string]interface{}{
+			"api_version":            3,
+			"auth":                   1,
+			"last_refreshed_on_time": getLastRefreshedOnTime(s.db.ListHTTPStates()),
+		})
+	}
+}
+
+func joinInts(values []int64) string {
+	var result strings.Builder
+	for i, val := range values {
+		fmt.Fprintf(&result, "%d", val)
+		if i != len(values)-1 {
+			result.WriteString(",")
+		}
+	}
+	return result.String()
+}
+
+func feedGroups(db *storage.Storage) []*FeverFeedsGroup {
+	feeds := db.ListFeeds()
+
+	groupFeeds := make(map[int64][]int64)
+	for _, feed := range feeds {
+		if feed.FolderId == nil {
+			continue
+		}
+		groupFeeds[*feed.FolderId] = append(groupFeeds[*feed.FolderId], feed.Id)
+	}
+	result := make([]*FeverFeedsGroup, 0)
+	for groupId, feedIds := range groupFeeds {
+		result = append(result, &FeverFeedsGroup{
+			GroupID: groupId,
+			FeedIDs: joinInts(feedIds),
+		})
+	}
+	return result
+}
+
+func (s *Server) feverGroupsHandler(c *router.Context) {
+	folders := s.db.ListFolders()
+	groups := make([]*FeverGroup, len(folders))
+	for i, folder := range folders {
+		groups[i] = &FeverGroup{ID: folder.Id, Title: folder.Title}
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"groups":       groups,
+		"feeds_groups": feedGroups(s.db),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverFeedsHandler(c *router.Context) {
+	feeds := s.db.ListFeeds()
+	httpStates := s.db.ListHTTPStates()
+
+	feverFeeds := make([]*FeverFeed, len(feeds))
+	for i, feed := range feeds {
+		var lastUpdated int64
+		if state, ok := httpStates[feed.Id]; ok {
+			lastUpdated = state.LastRefreshed.Unix()
+		}
+		feverFeeds[i] = &FeverFeed{
+			ID:          feed.Id,
+			FaviconID:   feed.Id,
+			Title:       feed.Title,
+			Url:         feed.FeedLink,
+			SiteUrl:     feed.Link,
+			IsSpark:     0,
+			LastUpdated: lastUpdated,
+		}
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"feeds":        feverFeeds,
+		"feeds_groups": feedGroups(s.db),
+	}, getLastRefreshedOnTime(httpStates))
+}
+
+func (s *Server) feverFaviconsHandler(c *router.Context) {
+	feeds := s.db.ListFeeds()
+	favicons := make([]*FeverFavicon, len(feeds))
+	for i, feed := range feeds {
+		data := "data:image/gif;base64,R0lGODlhAQABAAAAACw="
+		if feed.HasIcon {
+			icon := s.db.GetFeed(feed.Id).Icon
+			data = fmt.Sprintf(
+				"data:%s;base64,%s",
+				http.DetectContentType(*icon),
+				base64.StdEncoding.EncodeToString(*icon),
+			)
+		}
+		favicons[i] = &FeverFavicon{ID: feed.Id, Data: data}
+	}
+
+	writeFeverJSON(c, map[string]interface{}{
+		"favicons": favicons,
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+// for memory pressure reasons, we only return a limited number of items
+// documented at https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md#items
+const listLimit = 50
+
+func (s *Server) feverItemsHandler(c *router.Context) {
+	filter := storage.ItemFilter{}
+	query := c.Req.URL.Query()
+
+	switch {
+	case query.Get("with_ids") != "":
+		ids := make([]int64, 0)
+		for _, idstr := range strings.Split(query.Get("with_ids"), ",") {
+			if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+				ids = append(ids, idnum)
+			}
+		}
+		filter.IDs = &ids
+	case query.Get("since_id") != "":
+		idstr := query.Get("since_id")
+		if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+			filter.SinceID = &idnum
+		}
+	case query.Get("max_id") != "":
+		idstr := query.Get("max_id")
+		if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
+			filter.MaxID = &idnum
+		}
+	}
+
+	items := s.db.ListItems(filter, listLimit, true, true)
+
+	feverItems := make([]FeverItem, len(items))
+	for i, item := range items {
+		date := item.Date
+		time := date.Unix()
+
+		isSaved := 0
+		if item.Status == storage.STARRED {
+			isSaved = 1
+		}
+		isRead := 0
+		if item.Status == storage.READ {
+			isRead = 1
+		}
+		feverItems[i] = FeverItem{
+			ID:        item.Id,
+			FeedID:    item.FeedId,
+			Title:     item.Title,
+			Author:    "",
+			HTML:      item.Content,
+			Url:       item.Link,
+			IsSaved:   isSaved,
+			IsRead:    isRead,
+			CreatedAt: time,
+		}
+	}
+
+	totalItems := s.db.CountItems(storage.ItemFilter{})
+
+	writeFeverJSON(c, map[string]interface{}{
+		"items":       feverItems,
+		"total_items": totalItems,
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverLinksHandler(c *router.Context) {
+	writeFeverJSON(c, map[string]interface{}{
+		"links": make([]interface{}, 0),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
+	status := storage.UNREAD
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"unread_item_ids": joinInts(itemIds),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverSavedItemIDsHandler(c *router.Context) {
+	status := storage.STARRED
+	itemIds := make([]int64, 0)
+
+	itemFilter := storage.ItemFilter{
+		Status: &status,
+	}
+	for {
+		items := s.db.ListItems(itemFilter, listLimit, true, false)
+		if len(items) == 0 {
+			break
+		}
+		for _, item := range items {
+			itemIds = append(itemIds, item.Id)
+		}
+		itemFilter.After = &items[len(items)-1].Id
+	}
+	writeFeverJSON(c, map[string]interface{}{
+		"saved_item_ids": joinInts(itemIds),
+	}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
+}
+
+func (s *Server) feverMarkHandler(c *router.Context) {
+	id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64)
+	if err != nil {
+		log.Print("invalid id:", err)
+		return
+	}
+
+	switch c.Req.Form.Get("mark") {
+	case "item":
+		var status storage.ItemStatus
+		switch c.Req.Form.Get("as") {
+		case "read":
+			status = storage.READ
+		case "unread":
+			status = storage.UNREAD
+		case "saved":
+			status = storage.STARRED
+		case "unsaved":
+			status = storage.READ
+		default:
+			c.Out.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		s.db.UpdateItemStatus(id, status)
+	case "feed":
+		if c.Req.Form.Get("as") != "read" {
+			c.Out.WriteHeader(http.StatusBadRequest)
+		}
+		markFilter := storage.MarkFilter{FeedID: &id}
+		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
+			before := time.Unix(x, 0)
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
+	case "group":
+		if c.Req.Form.Get("as") != "read" {
+			c.Out.WriteHeader(http.StatusBadRequest)
+		}
+		markFilter := storage.MarkFilter{FolderID: &id}
+		x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
+		if x > 0 {
+			before := time.Unix(x, 0)
+			markFilter.Before = &before
+		}
+		s.db.MarkItemsRead(markFilter)
+	default:
+		c.Out.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	c.JSON(http.StatusOK, map[string]interface{}{
+		"api_version": 3,
+		"auth":        1,
+	})
+}

src/server/gzip/middleware.go

@@ -28,15 +28,16 @@ func (rw *gzipResponseWriter) WriteHeader(statusCode int) {
 }
 
 func Middleware(c *router.Context) {
-	if strings.Contains(c.Req.Header.Get("Accept-Encoding"), "gzip") {
-		gz := &gzipResponseWriter{out: gzip.NewWriter(c.Out), src: c.Out}
-		defer gz.out.Close()
-
-		c.Out.Header().Set("Content-Encoding", "gzip")
-		c.Out = gz
+	if !strings.Contains(c.Req.Header.Get("Accept-Encoding"), "gzip") {
 		c.Next()
-
 		return
 	}
+
+	gz := &gzipResponseWriter{out: gzip.NewWriter(c.Out), src: c.Out}
+	defer gz.out.Close()
+
+	c.Out.Header().Set("Content-Encoding", "gzip")
+	c.Out = gz
+
 	c.Next()
 }

src/server/opml/read.go

@@ -3,6 +3,8 @@ package opml
 import (
 	"encoding/xml"
 	"io"
+
+	"golang.org/x/net/html/charset"
 )
 
 type opml struct {
@@ -13,6 +15,7 @@ type opml struct {
 type outline struct {
 	Type     string    `xml:"type,attr,omitempty"`
 	Title    string    `xml:"text,attr"`
+	Title2   string    `xml:"title,attr,omitempty"`
 	FeedUrl  string    `xml:"xmlUrl,attr,omitempty"`
 	SiteUrl  string    `xml:"htmlUrl,attr,omitempty"`
 	Outlines []outline `xml:"outline,omitempty"`
@@ -21,14 +24,18 @@ type outline struct {
 func buildFolder(title string, outlines []outline) Folder {
 	folder := Folder{Title: title}
 	for _, outline := range outlines {
-		if outline.Type == "rss" {
+		if outline.Type == "rss" || outline.FeedUrl != "" {
 			folder.Feeds = append(folder.Feeds, Feed{
 				Title:   outline.Title,
 				FeedUrl: outline.FeedUrl,
 				SiteUrl: outline.SiteUrl,
 			})
 		} else {
-			subfolder := buildFolder(outline.Title, outline.Outlines)
+			title := outline.Title
+			if title == "" {
+				title = outline.Title2
+			}
+			subfolder := buildFolder(title, outline.Outlines)
 			folder.Folders = append(folder.Folders, subfolder)
 		}
 	}
@@ -40,6 +47,7 @@ func Parse(r io.Reader) (Folder, error) {
 	decoder := xml.NewDecoder(r)
 	decoder.Entity = xml.HTMLEntity
 	decoder.Strict = false
+	decoder.CharsetReader = charset.NewReaderLabel
 
 	err := decoder.Decode(&val)
 	if err != nil {

src/server/opml/read_test.go

@@ -1,6 +1,7 @@
 package opml
 
 import (
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -56,3 +57,72 @@ func TestParse(t *testing.T) {
 		t.Fatal("invalid opml")
 	}
 }
+
+func TestParseFallback(t *testing.T) {
+	// as reported in https://github.com/nkanaev/yarr/pull/56
+	// the feed below comes without `outline[text]` & `outline[type=rss]` attributes
+	have, _ := Parse(strings.NewReader(`
+		<?xml version="1.0" encoding="utf-8"?>
+		<opml xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" version="1.0">
+		  <head>
+			<title>Newsflow</title>
+		  </head>
+		  <body>
+			<outline title="foldertitle">
+				<outline htmlUrl="https://example.com" text="feedtext" title="feedtitle" xmlUrl="https://example.com/feed.xml" />
+			</outline>
+		  </body>
+		</opml>
+	`))
+	want := Folder{
+		Folders: []Folder{{
+			Title: "foldertitle",
+			Feeds: []Feed{
+				{Title: "feedtext", FeedUrl: "https://example.com/feed.xml", SiteUrl: "https://example.com"},
+			},
+		}},
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid opml")
+	}
+}
+
+func TestParseWithEncoding(t *testing.T) {
+	file, err := os.Open("sample_win1251.xml")
+	if err != nil {
+		t.Fatal(err)
+	}
+	have, err := Parse(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := Folder{
+		Title: "",
+		Feeds: []Feed{
+			{
+				Title:   "пример1",
+				FeedUrl: "https://baz.com/feed.xml",
+				SiteUrl: "https://baz.com/",
+			},
+		},
+		Folders: []Folder{
+			{
+				Title: "папка",
+				Feeds: []Feed{
+					{
+						Title:   "пример2",
+						FeedUrl: "https://foo.com/feed.xml",
+						SiteUrl: "https://foo.com/",
+					},
+				},
+			},
+		},
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid opml")
+	}
+}

src/server/opml/sample_win1251.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="windows-1251"?>
+<opml version="1.1">
+<head><title><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD></title></head>
+<body>
+    <outline text="<22><><EFBFBD><EFBFBD><EFBFBD>">
+        <outline type="rss" text="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2" description="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>2" xmlUrl="https://foo.com/feed.xml" htmlUrl="https://foo.com/"/>
+    </outline>
+    <outline type="rss" text="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1" description="<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1" xmlUrl="https://baz.com/feed.xml" htmlUrl="https://baz.com/"/>
+</body>
+</opml>

src/server/router/context.go

@@ -29,15 +29,15 @@ func (c *Context) JSON(status int, data interface{}) {
 	if err != nil {
 		log.Fatal(err)
 	}
-	c.Out.WriteHeader(status)
 	c.Out.Header().Set("Content-Type", "application/json; charset=utf-8")
+	c.Out.WriteHeader(status)
 	c.Out.Write(body)
 	c.Out.Write([]byte("\n"))
 }
 
 func (c *Context) HTML(status int, tmpl *template.Template, data interface{}) {
-	c.Out.WriteHeader(status)
 	c.Out.Header().Set("Content-Type", "text/html")
+	c.Out.WriteHeader(status)
 	tmpl.Execute(c.Out, data)
 }
 

src/server/router/router.go

@@ -32,10 +32,13 @@ func (r *Router) Use(h Handler) {
 }
 
 func (r *Router) For(path string, handler Handler) {
+	chain := make([]Handler, 0)
+	chain = append(chain, r.middle...)
+	chain = append(chain, handler)
+
 	x := Route{}
 	x.regex = routeRegexp(path)
-	x.chain = append(r.middle, handler)
-
+	x.chain = chain
 	r.routes = append(r.routes, x)
 }
 

src/server/routes.go

@@ -1,15 +1,18 @@
 package server
 
 import (
+	"crypto/md5"
 	"encoding/json"
+	"fmt"
 	"log"
-	"math"
 	"net/http"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 
 	"github.com/nkanaev/yarr/src/assets"
+	"github.com/nkanaev/yarr/src/content/htmlutil"
 	"github.com/nkanaev/yarr/src/content/readability"
 	"github.com/nkanaev/yarr/src/content/sanitizer"
 	"github.com/nkanaev/yarr/src/content/silo"
@@ -31,12 +34,14 @@ func (s *Server) handler() http.Handler {
 			BasePath: s.BasePath,
 			Username: s.Username,
 			Password: s.Password,
-			Public:   "/static",
+			Public:   []string{"/static", "/fever"},
+			DB:       s.db,
 		}
 		r.Use(a.Handler)
 	}
 
 	r.For("/", s.handleIndex)
+	r.For("/manifest.json", s.handleManifest)
 	r.For("/static/*path", s.handleStatic)
 	r.For("/api/status", s.handleStatus)
 	r.For("/api/folders", s.handleFolderList)
@@ -53,6 +58,7 @@ func (s *Server) handler() http.Handler {
 	r.For("/opml/export", s.handleOPMLExport)
 	r.For("/page", s.handlePageCrawl)
 	r.For("/logout", s.handleLogout)
+	r.For("/fever/", s.handleFever)
 
 	return r
 }
@@ -74,6 +80,24 @@ func (s *Server) handleStatic(c *router.Context) {
 	http.StripPrefix(s.BasePath+"/static/", http.FileServer(http.FS(assets.FS))).ServeHTTP(c.Out, c.Req)
 }
 
+func (s *Server) handleManifest(c *router.Context) {
+	c.JSON(http.StatusOK, map[string]interface{}{
+		"$schema":     "https://json.schemastore.org/web-manifest-combined.json",
+		"name":        "yarr!",
+		"short_name":  "yarr",
+		"description": "yet another rss reader",
+		"display":     "standalone",
+		"start_url":   "/" + strings.TrimPrefix(s.BasePath, "/"),
+		"icons": []map[string]interface{}{
+			{
+				"src":   s.BasePath + "/static/graphicarts/favicon.png",
+				"sizes": "64x64",
+				"type":  "image/png",
+			},
+		},
+	})
+}
+
 func (s *Server) handleStatus(c *router.Context) {
 	c.JSON(http.StatusOK, map[string]interface{}{
 		"running": s.worker.FeedsPending(),
@@ -143,19 +167,55 @@ func (s *Server) handleFeedErrors(c *router.Context) {
 	c.JSON(http.StatusOK, errors)
 }
 
+type feedicon struct {
+	ctype string
+	bytes []byte
+	etag  string
+}
+
 func (s *Server) handleFeedIcon(c *router.Context) {
 	id, err := c.VarInt64("id")
 	if err != nil {
 		c.Out.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	feed := s.db.GetFeed(id)
-	if feed != nil && feed.Icon != nil {
-		c.Out.Header().Set("Content-Type", http.DetectContentType(*feed.Icon))
-		c.Out.Write(*feed.Icon)
-	} else {
-		c.Out.WriteHeader(http.StatusNotFound)
+
+	cachekey := "icon:" + strconv.FormatInt(id, 10)
+	s.cache_mutex.Lock()
+	cachedat := s.cache[cachekey]
+	s.cache_mutex.Unlock()
+	if cachedat == nil {
+		feed := s.db.GetFeed(id)
+		if feed == nil || feed.Icon == nil {
+			c.Out.WriteHeader(http.StatusNotFound)
+			return
+		}
+
+		hash := md5.New()
+		hash.Write(*feed.Icon)
+
+		etag := fmt.Sprintf("%x", hash.Sum(nil))[:16]
+
+		cachedat = feedicon{
+			ctype: http.DetectContentType(*feed.Icon),
+			bytes: *(*feed).Icon,
+			etag:  etag,
+		}
+		s.cache_mutex.Lock()
+		s.cache[cachekey] = cachedat
+		s.cache_mutex.Unlock()
 	}
+
+	icon := cachedat.(feedicon)
+
+	if c.Req.Header.Get("If-None-Match") == icon.etag {
+		c.Out.WriteHeader(http.StatusNotModified)
+		return
+	}
+
+	c.Out.Header().Set("Content-Type", icon.ctype)
+	c.Out.Header().Set("Etag", icon.etag)
+	c.Out.Write(icon.bytes)
 }
 
 func (s *Server) handleFeedList(c *router.Context) {
@@ -185,12 +245,17 @@ func (s *Server) handleFeedList(c *router.Context) {
 				result.FeedLink,
 				form.FolderID,
 			)
-			s.db.CreateItems(worker.ConvertItems(result.Feed.Items, *feed))
+			items := worker.ConvertItems(result.Feed.Items, *feed)
+			if len(items) > 0 {
+				s.db.CreateItems(items)
+				s.db.SetFeedSize(feed.Id, len(items))
+				s.db.SyncSearch()
+			}
 			s.worker.FindFeedFavicon(*feed)
 
 			c.JSON(http.StatusOK, map[string]interface{}{
 				"status": "success",
-				"feed": feed,
+				"feed":   feed,
 			})
 		default:
 			c.JSON(http.StatusOK, map[string]string{"status": "notfound"})
@@ -229,6 +294,11 @@ func (s *Server) handleFeed(c *router.Context) {
 				s.db.UpdateFeedFolder(id, &folderId)
 			}
 		}
+		if link, ok := body["feed_link"]; ok {
+			if reflect.TypeOf(link).Kind() == reflect.String {
+				s.db.UpdateFeedLink(id, link.(string))
+			}
+		}
 		c.Out.WriteHeader(http.StatusOK)
 	} else if c.Req.Method == "DELETE" {
 		s.db.DeleteFeed(id)
@@ -250,7 +320,18 @@ func (s *Server) handleItem(c *router.Context) {
 			c.Out.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
+		// runtime fix for relative links
+		if !htmlutil.IsAPossibleLink(item.Link) {
+			if feed := s.db.GetFeed(item.FeedId); feed != nil {
+				item.Link = htmlutil.AbsoluteUrl(item.Link, feed.Link)
+			}
+		}
+
 		item.Content = sanitizer.Sanitize(item.Link, item.Content)
+		for i, link := range item.MediaLinks {
+			item.MediaLinks[i].Description = sanitizer.Sanitize(item.Link, link.Description)
+		}
 
 		c.JSON(http.StatusOK, item)
 	} else if c.Req.Method == "PUT" {
@@ -272,11 +353,8 @@ func (s *Server) handleItem(c *router.Context) {
 func (s *Server) handleItemList(c *router.Context) {
 	if c.Req.Method == "GET" {
 		perPage := 20
-		curPage := 1
 		query := c.Req.URL.Query()
-		if page, err := c.QueryInt64("page"); err == nil {
-			curPage = int(page)
-		}
+
 		filter := storage.ItemFilter{}
 		if folderID, err := c.QueryInt64("folder_id"); err == nil {
 			filter.FolderID = &folderID
@@ -284,6 +362,9 @@ func (s *Server) handleItemList(c *router.Context) {
 		if feedID, err := c.QueryInt64("feed_id"); err == nil {
 			filter.FeedID = &feedID
 		}
+		if after, err := c.QueryInt64("after"); err == nil {
+			filter.After = &after
+		}
 		if status := query.Get("status"); len(status) != 0 {
 			statusValue := storage.StatusValues[status]
 			filter.Status = &statusValue
@@ -292,14 +373,23 @@ func (s *Server) handleItemList(c *router.Context) {
 			filter.Search = &search
 		}
 		newestFirst := query.Get("oldest_first") != "true"
-		items := s.db.ListItems(filter, (curPage-1)*perPage, perPage, newestFirst)
-		count := s.db.CountItems(filter)
+
+		items := s.db.ListItems(filter, perPage+1, newestFirst, true)
+		hasMore := false
+		if len(items) == perPage+1 {
+			hasMore = true
+			items = items[:perPage]
+		}
+
+		for i, item := range items {
+			if item.Title == "" {
+				text := htmlutil.ExtractText(item.Content)
+				items[i].Title = htmlutil.TruncateText(text, 140)
+			}
+		}
 		c.JSON(http.StatusOK, map[string]interface{}{
-			"page": map[string]int{
-				"cur": curPage,
-				"num": int(math.Ceil(float64(count) / float64(perPage))),
-			},
-			"list": items,
+			"list":     items,
+			"has_more": hasMore,
 		})
 	} else if c.Req.Method == "PUT" {
 		filter := storage.MarkFilter{}
@@ -414,24 +504,27 @@ func (s *Server) handleOPMLExport(c *router.Context) {
 func (s *Server) handlePageCrawl(c *router.Context) {
 	url := c.Req.URL.Query().Get("url")
 
+	if newUrl := silo.RedirectURL(url); newUrl != "" {
+		url = newUrl
+	}
 	if content := silo.VideoIFrame(url); content != "" {
 		c.JSON(http.StatusOK, map[string]string{
-			"content": content,
+			"content": sanitizer.Sanitize(url, content),
 		})
 		return
 	}
 
-	res, err := http.Get(url)
+	body, err := worker.GetBody(url)
 	if err != nil {
 		log.Print(err)
 		c.Out.WriteHeader(http.StatusBadRequest)
 		return
 	}
-	defer res.Body.Close()
-	content, err := readability.ExtractContent(res.Body)
+	content, err := readability.ExtractContent(strings.NewReader(body))
 	if err != nil {
-		log.Print(err)
-		c.Out.WriteHeader(http.StatusNoContent)
+		c.JSON(http.StatusOK, map[string]string{
+			"content": "error: " + err.Error(),
+		})
 		return
 	}
 	content = sanitizer.Sanitize(url, content)

src/server/routes_test.go

@@ -1,8 +1,16 @@
 package server
 
 import (
+	"fmt"
+	"io"
+	"log"
+	"net/http"
 	"net/http/httptest"
+	"os"
+	"reflect"
 	"testing"
+
+	"github.com/nkanaev/yarr/src/storage"
 )
 
 func TestStatic(t *testing.T) {
@@ -43,3 +51,64 @@ func TestStaticBanTemplates(t *testing.T) {
 		t.FailNow()
 	}
 }
+
+func TestIndexGzipped(t *testing.T) {
+	log.SetOutput(io.Discard)
+	db, _ := storage.New(":memory:")
+	log.SetOutput(os.Stderr)
+	handler := NewServer(db, "127.0.0.1:8000").handler()
+	url := "/"
+
+	recorder := httptest.NewRecorder()
+	request := httptest.NewRequest("GET", url, nil)
+	request.Header.Set("accept-encoding", "gzip")
+	handler.ServeHTTP(recorder, request)
+	response := recorder.Result()
+	if response.StatusCode != 200 {
+		t.FailNow()
+	}
+	if response.Header.Get("content-encoding") != "gzip" {
+		t.Errorf("invalid content-encoding header: %#v", response.Header.Get("content-encoding"))
+	}
+	if response.Header.Get("content-type") != "text/html" {
+		t.Errorf("invalid content-type header: %#v", response.Header.Get("content-type"))
+	}
+}
+
+func TestFeedIcons(t *testing.T) {
+	log.SetOutput(io.Discard)
+	db, _ := storage.New(":memory:")
+	icon := []byte("test")
+	feed := db.CreateFeed("", "", "", "", nil)
+	db.UpdateFeedIcon(feed.Id, &icon)
+	log.SetOutput(os.Stderr)
+
+	recorder := httptest.NewRecorder()
+	url := fmt.Sprintf("/api/feeds/%d/icon", feed.Id)
+	request := httptest.NewRequest("GET", url, nil)
+
+	handler := NewServer(db, "127.0.0.1:8000").handler()
+	handler.ServeHTTP(recorder, request)
+	response := recorder.Result()
+
+	if response.StatusCode != http.StatusOK {
+		t.Fatal()
+	}
+	body, _ := io.ReadAll(response.Body)
+	if !reflect.DeepEqual(body, icon) {
+		t.Fatal()
+	}
+	if response.Header.Get("Etag") == "" {
+		t.Fatal()
+	}
+
+	recorder2 := httptest.NewRecorder()
+	request2 := httptest.NewRequest("GET", url, nil)
+	request2.Header.Set("If-None-Match", response.Header.Get("Etag"))
+	handler.ServeHTTP(recorder2, request2)
+	response2 := recorder2.Result()
+
+	if response2.StatusCode != http.StatusNotModified {
+		t.Fatal("got", response2.StatusCode)
+	}
+}

src/server/server.go

@@ -3,15 +3,18 @@ package server
 import (
 	"log"
 	"net/http"
+	"sync"
 
 	"github.com/nkanaev/yarr/src/storage"
 	"github.com/nkanaev/yarr/src/worker"
 )
 
 type Server struct {
-	Addr   string
-	db     *storage.Storage
-	worker *worker.Worker
+	Addr        string
+	db          *storage.Storage
+	worker      *worker.Worker
+	cache       map[string]interface{}
+	cache_mutex *sync.Mutex
 
 	BasePath string
 
@@ -25,9 +28,11 @@ type Server struct {
 
 func NewServer(db *storage.Storage, addr string) *Server {
 	return &Server{
-		db:     db,
-		Addr:   addr,
-		worker: worker.NewWorker(db),
+		db:          db,
+		Addr:        addr,
+		worker:      worker.NewWorker(db),
+		cache:       make(map[string]interface{}),
+		cache_mutex: &sync.Mutex{},
 	}
 }
 

src/storage/feed.go

@@ -20,18 +20,19 @@ func (s *Storage) CreateFeed(title, description, link, feedLink string, folderId
 	if title == "" {
 		title = feedLink
 	}
-	result, err := s.db.Exec(`
+	row := s.db.QueryRow(`
 		insert into feeds (title, description, link, feed_link, folder_id) 
 		values (?, ?, ?, ?, ?)
-		on conflict (feed_link) do update set folder_id=?`,
+		on conflict (feed_link) do update set folder_id = ?
+        returning id`,
 		title, description, link, feedLink, folderId,
 		folderId,
 	)
+
+	var id int64
+	err := row.Scan(&id)
 	if err != nil {
-		return nil
-	}
-	id, idErr := result.LastInsertId()
-	if idErr != nil {
+		log.Print(err)
 		return nil
 	}
 	return &Feed{
@@ -70,16 +71,21 @@ func (s *Storage) UpdateFeedFolder(feedId int64, newFolderId *int64) bool {
 	return err == nil
 }
 
+func (s *Storage) UpdateFeedLink(feedId int64, newLink string) bool {
+	_, err := s.db.Exec(`update feeds set feed_link = ? where id = ?`, newLink, feedId)
+	return err == nil
+}
+
 func (s *Storage) UpdateFeedIcon(feedId int64, icon *[]byte) bool {
 	_, err := s.db.Exec(`update feeds set icon = ? where id = ?`, icon, feedId)
 	return err == nil
 }
 
 func (s *Storage) ListFeeds() []Feed {
-	result := make([]Feed, 0, 0)
+	result := make([]Feed, 0)
 	rows, err := s.db.Query(`
 		select id, folder_id, title, description, link, feed_link,
-		       ifnull(icon, '') != '' as has_icon
+		       ifnull(length(icon), 0) > 0 as has_icon
 		from feeds
 		order by title collate nocase
 	`)
@@ -107,6 +113,36 @@ func (s *Storage) ListFeeds() []Feed {
 	return result
 }
 
+func (s *Storage) ListFeedsMissingIcons() []Feed {
+	result := make([]Feed, 0)
+	rows, err := s.db.Query(`
+		select id, folder_id, title, description, link, feed_link
+		from feeds
+		where icon is null
+	`)
+	if err != nil {
+		log.Print(err)
+		return result
+	}
+	for rows.Next() {
+		var f Feed
+		err = rows.Scan(
+			&f.Id,
+			&f.FolderId,
+			&f.Title,
+			&f.Description,
+			&f.Link,
+			&f.FeedLink,
+		)
+		if err != nil {
+			log.Print(err)
+			return result
+		}
+		result = append(result, f)
+	}
+	return result
+}
+
 func (s *Storage) GetFeed(id int64) *Feed {
 	var f Feed
 	err := s.db.QueryRow(`
@@ -164,3 +200,15 @@ func (s *Storage) GetFeedErrors() map[int64]string {
 	}
 	return errors
 }
+
+func (s *Storage) SetFeedSize(feedId int64, size int) {
+	_, err := s.db.Exec(`
+		insert into feed_sizes (feed_id, size)
+		values (?, ?)
+		on conflict (feed_id) do update set size = excluded.size`,
+		feedId, size,
+	)
+	if err != nil {
+		log.Print(err)
+	}
+}

src/storage/feed_test.go

@@ -17,6 +17,23 @@ func TestCreateFeed(t *testing.T) {
 	}
 }
 
+func TestCreateFeedSameLink(t *testing.T) {
+	db := testDB()
+	feed1 := db.CreateFeed("title", "", "", "http://example1.com/feed.xml", nil)
+	if feed1 == nil || feed1.Id == 0 {
+		t.Fatal("expected feed")
+	}
+
+	for i := 0; i < 10; i++ {
+		db.CreateFeed("title", "", "", "http://example2.com/feed.xml", nil)
+	}
+
+	feed2 := db.CreateFeed("title", "", "http://example.com", "http://example1.com/feed.xml", nil)
+	if feed1.Id != feed2.Id {
+		t.Fatalf("expected the same feed.\nwant: %#v\nhave: %#v", feed1, feed2)
+	}
+}
+
 func TestReadFeed(t *testing.T) {
 	db := testDB()
 	if db.GetFeed(100500) != nil {

src/storage/folder.go

@@ -1,7 +1,6 @@
 package storage
 
 import (
-	"fmt"
 	"log"
 )
 
@@ -13,35 +12,21 @@ type Folder struct {
 
 func (s *Storage) CreateFolder(title string) *Folder {
 	expanded := true
-	result, err := s.db.Exec(`
+	row := s.db.QueryRow(`
 		insert into folders (title, is_expanded) values (?, ?)
-		on conflict (title) do nothing`,
+		on conflict (title) do update set title = ?
+        returning id`,
 		title, expanded,
+		// provide title again so that we can extract row id
+		title,
 	)
-	if err != nil {
-		fmt.Println(err)
-		return nil
-	}
-
 	var id int64
-	numrows, err := result.RowsAffected()
+	err := row.Scan(&id)
+
 	if err != nil {
 		log.Print(err)
 		return nil
 	}
-	if numrows == 1 {
-		id, err = result.LastInsertId()
-		if err != nil {
-			log.Print(err)
-			return nil
-		}
-	} else {
-		err = s.db.QueryRow(`select id, is_expanded from folders where title=?`, title).Scan(&id, &expanded)
-		if err != nil {
-			log.Print(err)
-			return nil
-		}
-	}
 	return &Folder{Id: id, Title: title, IsExpanded: expanded}
 }
 

src/storage/item.go

@@ -1,9 +1,11 @@
 package storage
 
 import (
+	"database/sql/driver"
 	"encoding/json"
 	"fmt"
 	"log"
+	"sort"
 	"strings"
 	"time"
 
@@ -43,17 +45,35 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
+type MediaLink struct {
+	URL         string `json:"url"`
+	Type        string `json:"type"`
+	Description string `json:"description,omitempty"`
+}
+
+type MediaLinks []MediaLink
+
+func (m *MediaLinks) Scan(src any) error {
+	if data, ok := src.([]byte); ok {
+		return json.Unmarshal(data, m)
+	}
+	return nil
+}
+
+func (m MediaLinks) Value() (driver.Value, error) {
+	return json.Marshal(m)
+}
+
 type Item struct {
-	Id       int64      `json:"id"`
-	GUID     string     `json:"guid"`
-	FeedId   int64      `json:"feed_id"`
-	Title    string     `json:"title"`
-	Link     string     `json:"link"`
-	Content  string     `json:"content,omitempty"`
-	Date     time.Time  `json:"date"`
-	Status   ItemStatus `json:"status"`
-	ImageURL *string    `json:"image"`
-	AudioURL *string    `json:"podcast_url"`
+	Id         int64      `json:"id"`
+	GUID       string     `json:"guid"`
+	FeedId     int64      `json:"feed_id"`
+	Title      string     `json:"title"`
+	Link       string     `json:"link"`
+	Content    string     `json:"content,omitempty"`
+	Date       time.Time  `json:"date"`
+	Status     ItemStatus `json:"status"`
+	MediaLinks MediaLinks `json:"media_links"`
 }
 
 type ItemFilter struct {
@@ -61,11 +81,36 @@ type ItemFilter struct {
 	FeedID   *int64
 	Status   *ItemStatus
 	Search   *string
+	After    *int64
+	IDs      *[]int64
+	SinceID  *int64
+	MaxID    *int64
+	Before   *time.Time
 }
 
 type MarkFilter struct {
 	FolderID *int64
 	FeedID   *int64
+
+	Before *time.Time
+}
+
+type ItemList []Item
+
+func (list ItemList) Len() int {
+	return len(list)
+}
+
+func (list ItemList) SortKey(i int) string {
+	return list[i].Date.Format(time.RFC3339) + "::" + list[i].GUID
+}
+
+func (list ItemList) Less(i, j int) bool {
+	return list.SortKey(i) < list.SortKey(j)
+}
+
+func (list ItemList) Swap(i, j int) {
+	list[i], list[j] = list[j], list[i]
 }
 
 func (s *Storage) CreateItems(items []Item) bool {
@@ -75,19 +120,26 @@ func (s *Storage) CreateItems(items []Item) bool {
 		return false
 	}
 
-	now := time.Now()
+	now := time.Now().UTC()
 
-	for _, item := range items {
+	itemsSorted := ItemList(items)
+	sort.Sort(itemsSorted)
+
+	for _, item := range itemsSorted {
 		_, err = tx.Exec(`
 			insert into items (
 				guid, feed_id, title, link, date,
-				content, image, podcast_url,
+				content, media_links,
 				date_arrived, status
 			)
-			values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+			values (
+				?, ?, ?, ?, strftime('%Y-%m-%d %H:%M:%f', ?),
+				?, ?,
+				?, ?
+			)
 			on conflict (feed_id, guid) do nothing`,
 			item.GUID, item.FeedId, item.Title, item.Link, item.Date,
-			item.Content, item.ImageURL, item.AudioURL,
+			item.Content, item.MediaLinks,
 			now, UNREAD,
 		)
 		if err != nil {
@@ -106,7 +158,7 @@ func (s *Storage) CreateItems(items []Item) bool {
 	return true
 }
 
-func listQueryPredicate(filter ItemFilter) (string, []interface{}) {
+func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interface{}) {
 	cond := make([]string, 0)
 	args := make([]interface{}, 0)
 	if filter.FolderID != nil {
@@ -131,6 +183,36 @@ func listQueryPredicate(filter ItemFilter) (string, []interface{}) {
 		cond = append(cond, "i.search_rowid in (select rowid from search where search match ?)")
 		args = append(args, strings.Join(terms, " "))
 	}
+	if filter.After != nil {
+		compare := ">"
+		if newestFirst {
+			compare = "<"
+		}
+		cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = ?)", compare))
+		args = append(args, *filter.After)
+	}
+	if filter.IDs != nil && len(*filter.IDs) > 0 {
+		qmarks := make([]string, len(*filter.IDs))
+		idargs := make([]interface{}, len(*filter.IDs))
+		for i, id := range *filter.IDs {
+			qmarks[i] = "?"
+			idargs[i] = id
+		}
+		cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")")
+		args = append(args, idargs...)
+	}
+	if filter.SinceID != nil {
+		cond = append(cond, "i.id > ?")
+		args = append(args, filter.SinceID)
+	}
+	if filter.MaxID != nil {
+		cond = append(cond, "i.id < ?")
+		args = append(args, filter.MaxID)
+	}
+	if filter.Before != nil {
+		cond = append(cond, "i.date < ?")
+		args = append(args, filter.Before)
+	}
 
 	predicate := "1"
 	if len(cond) > 0 {
@@ -140,25 +222,51 @@ func listQueryPredicate(filter ItemFilter) (string, []interface{}) {
 	return predicate, args
 }
 
-func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bool) []Item {
-	predicate, args := listQueryPredicate(filter)
+func (s *Storage) CountItems(filter ItemFilter) int {
+	predicate, args := listQueryPredicate(filter, false)
+
+	var count int
+	query := fmt.Sprintf(`
+		select count(*)
+		from items
+		where %s
+		`, predicate)
+	err := s.db.QueryRow(query, args...).Scan(&count)
+	if err != nil {
+		log.Print(err)
+		return 0
+	}
+	return count
+}
+
+func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item {
+	predicate, args := listQueryPredicate(filter, newestFirst)
 	result := make([]Item, 0, 0)
 
-	order := "date desc"
+	order := "date desc, id desc"
 	if !newestFirst {
-		order = "date asc"
+		order = "date asc, id asc"
+	}
+	if filter.IDs != nil || filter.SinceID != nil {
+		order = "i.id asc"
+	}
+	if filter.MaxID != nil {
+		order = "i.id desc"
 	}
 
+	selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.media_links"
+	if withContent {
+		selectCols += ", i.content"
+	} else {
+		selectCols += ", '' as content"
+	}
 	query := fmt.Sprintf(`
-		select
-			i.id, i.guid, i.feed_id,
-			i.title, i.link, i.date,
-			i.status, i.image, i.podcast_url
+		select %s
 		from items i
 		where %s
 		order by %s
-		limit %d offset %d
-		`, predicate, order, limit, offset)
+		limit %d
+		`, selectCols, predicate, order, limit)
 	rows, err := s.db.Query(query, args...)
 	if err != nil {
 		log.Print(err)
@@ -169,7 +277,7 @@ func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bo
 		err = rows.Scan(
 			&x.Id, &x.GUID, &x.FeedId,
 			&x.Title, &x.Link, &x.Date,
-			&x.Status, &x.ImageURL, &x.AudioURL,
+			&x.Status, &x.MediaLinks, &x.Content,
 		)
 		if err != nil {
 			log.Print(err)
@@ -185,12 +293,12 @@ func (s *Storage) GetItem(id int64) *Item {
 	err := s.db.QueryRow(`
 		select
 			i.id, i.guid, i.feed_id, i.title, i.link, i.content,
-			i.date, i.status, i.image, i.podcast_url
+			i.date, i.status, i.media_links
 		from items i
 		where i.id = ?
 	`, id).Scan(
 		&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
-		&i.Date, &i.Status, &i.ImageURL, &i.AudioURL,
+		&i.Date, &i.Status, &i.MediaLinks,
 	)
 	if err != nil {
 		log.Print(err)
@@ -199,28 +307,17 @@ func (s *Storage) GetItem(id int64) *Item {
 	return i
 }
 
-func (s *Storage) CountItems(filter ItemFilter) int64 {
-	predicate, args := listQueryPredicate(filter)
-	query := fmt.Sprintf(`
-		select count(i.id)
-		from items i
-		where %s`, predicate)
-	row := s.db.QueryRow(query, args...)
-	if row != nil {
-		var result int64
-		row.Scan(&result)
-		return result
-	}
-	return 0
-}
-
 func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
 	_, err := s.db.Exec(`update items set status = ? where id = ?`, status, item_id)
 	return err == nil
 }
 
 func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
-	predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID})
+	predicate, args := listQueryPredicate(ItemFilter{
+		FolderID: filter.FolderID,
+		FeedID:   filter.FeedID,
+		Before:   filter.Before,
+	}, false)
 	query := fmt.Sprintf(`
 		update items as i set status = %d
 		where %s and i.status != %d
@@ -298,45 +395,70 @@ func (s *Storage) SyncSearch() {
 	}
 }
 
+var (
+	itemsKeepSize = 50
+	itemsKeepDays = 90
+)
+
+// Delete old articles from the database to cleanup space.
+//
+// The rules:
+//   - Never delete starred entries.
+//   - Keep at least the same amount of articles the feed provides (default: 50).
+//     This prevents from deleting items for rarely updated and/or ever-growing
+//     feeds which might eventually reappear as unread.
+//   - Keep entries for a certain period (default: 90 days).
 func (s *Storage) DeleteOldItems() {
-	rows, err := s.db.Query(fmt.Sprintf(`
-		select feed_id, count(*) as num_items
-		from items
-		where status != %d
-		group by feed_id
-		having num_items > 50
-	`, STARRED))
+	rows, err := s.db.Query(`
+		select
+			i.feed_id,
+			max(coalesce(s.size, 0), ?) as max_items,
+			count(*) as num_items
+		from items i
+		left outer join feed_sizes s on s.feed_id = i.feed_id
+		where status != ?
+		group by i.feed_id
+	`, itemsKeepSize, STARRED)
 
 	if err != nil {
 		log.Print(err)
 		return
 	}
 
-	feedIds := make([]int64, 0)
+	feedLimits := make(map[int64]int64, 0)
 	for rows.Next() {
-		var id int64
-		rows.Scan(&id, nil)
-		feedIds = append(feedIds, id)
+		var feedId, limit int64
+		rows.Scan(&feedId, &limit, nil)
+		feedLimits[feedId] = limit
 	}
 
-	for _, feedId := range feedIds {
+	for feedId, limit := range feedLimits {
 		result, err := s.db.Exec(`
-			delete from items where feed_id = ? and status != ? and date_arrived < ?`,
+			delete from items
+			where id in (
+				select i.id
+				from items i
+				where i.feed_id = ? and status != ?
+				order by date desc
+				limit -1 offset ?
+			) and date_arrived < ?
+			`,
 			feedId,
 			STARRED,
-			time.Now().Add(-time.Hour*24*90), // 90 days
+			limit,
+			time.Now().UTC().Add(-time.Hour*time.Duration(24*itemsKeepDays)),
 		)
 		if err != nil {
 			log.Print(err)
 			return
 		}
-		num, err := result.RowsAffected()
+		numDeleted, err := result.RowsAffected()
 		if err != nil {
 			log.Print(err)
 			return
 		}
-		if num > 0 {
-			log.Printf("Deleted %d old items (%d)", num, feedId)
+		if numDeleted > 0 {
+			log.Printf("Deleted %d old items (feed: %d)", numDeleted, feedId)
 		}
 	}
 }

src/storage/item_test.go

@@ -1,7 +1,9 @@
 package storage
 
 import (
+	"log"
 	"reflect"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -44,18 +46,18 @@ func testItemsSetup(db *Storage) testItemScope {
 	db.CreateItems([]Item{
 		// feed11
 		{GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)},
-		{GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)},
-		{GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)},
+		{GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)}, // read
+		{GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)}, // starred
 		// feed12
 		{GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)},
-		{GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)},
+		{GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)}, // read
 		// feed21
-		{GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)},
-		{GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)},
+		{GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)}, // read
+		{GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)}, // starred
 		// feed01
 		{GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)},
-		{GUID: "item012", FeedId: feed01.Id, Title: "title012", Date: now.Add(time.Hour * 24 * 9)},
-		{GUID: "item013", FeedId: feed01.Id, Title: "title013", Date: now.Add(time.Hour * 24 * 10)},
+		{GUID: "item012", FeedId: feed01.Id, Title: "title012", Date: now.Add(time.Hour * 24 * 9)},  // read
+		{GUID: "item013", FeedId: feed01.Id, Title: "title013", Date: now.Add(time.Hour * 24 * 10)}, // starred
 	})
 	db.db.Exec(`update items set status = ? where guid in ("item112", "item122", "item211", "item012")`, READ)
 	db.db.Exec(`update items set status = ? where guid in ("item113", "item212", "item013")`, STARRED)
@@ -70,7 +72,25 @@ func testItemsSetup(db *Storage) testItemScope {
 	}
 }
 
-func testItemGuids(items []Item) []string {
+func getItem(db *Storage, guid string) *Item {
+	i := &Item{}
+	err := db.db.QueryRow(`
+		select
+			i.id, i.guid, i.feed_id, i.title, i.link, i.content,
+			i.date, i.status, i.media_links
+		from items i
+		where i.guid = ?
+	`, guid).Scan(
+		&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
+		&i.Date, &i.Status, &i.MediaLinks,
+	)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return i
+}
+
+func getItemGuids(items []Item) []string {
 	guids := make([]string, 0)
 	for _, item := range items {
 		guids = append(guids, item.GUID)
@@ -84,7 +104,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by folder_id
 
-	have := testItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 0, 10, false))
+	have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
 	want := []string{"item111", "item112", "item113", "item121", "item122"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -92,7 +112,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = testItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
 	want = []string{"item211", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -102,7 +122,7 @@ func TestListItems(t *testing.T) {
 
 	// filter by feed_id
 
-	have = testItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
 	want = []string{"item111", "item112", "item113"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -110,7 +130,7 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = testItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
 	want = []string{"item011", "item012", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -121,7 +141,7 @@ func TestListItems(t *testing.T) {
 	// filter by status
 
 	var starred ItemStatus = STARRED
-	have = testItemGuids(db.ListItems(ItemFilter{Status: &starred}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
 	want = []string{"item113", "item212", "item013"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -130,7 +150,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	var unread ItemStatus = UNREAD
-	have = testItemGuids(db.ListItems(ItemFilter{Status: &unread}, 0, 10, false))
+	have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
 	want = []string{"item111", "item121", "item011"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -138,9 +158,9 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	// filter by offset,limit
+	// limit
 
-	have = testItemGuids(db.ListItems(ItemFilter{}, 0, 2, false))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
 	want = []string{"item111", "item112"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -148,18 +168,10 @@ func TestListItems(t *testing.T) {
 		t.Fail()
 	}
 
-	have = testItemGuids(db.ListItems(ItemFilter{}, 2, 3, false))
-	want = []string{"item113", "item121", "item122"}
-	if !reflect.DeepEqual(have, want) {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
 	// filter by search
 	db.SyncSearch()
 	search1 := "title111"
-	have = testItemGuids(db.ListItems(ItemFilter{Search: &search1}, 0, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
 	want = []string{"item111"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -168,7 +180,7 @@ func TestListItems(t *testing.T) {
 	}
 
 	// sort by date
-	have = testItemGuids(db.ListItems(ItemFilter{}, 0, 4, true))
+	have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
 	want = []string{"item013", "item012", "item011", "item212"}
 	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
@@ -177,72 +189,37 @@ func TestListItems(t *testing.T) {
 	}
 }
 
-func TestCountItems(t *testing.T) {
+func TestListItemsPaginated(t *testing.T) {
 	db := testDB()
-	scope := testItemsSetup(db)
+	testItemsSetup(db)
 
-	have := db.CountItems(ItemFilter{})
-	want := int64(10)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-	
-	// folders
+	item012 := getItem(db, "item012")
+	item121 := getItem(db, "item121")
 
-	have = db.CountItems(ItemFilter{FolderID: &scope.folder1.Id})
-	want = int64(5)
-	if have != want {
+	// all, newest first
+	have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
+	want := []string{"item011", "item212", "item211"}
+	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.Fail()
 	}
 
-	have = db.CountItems(ItemFilter{FolderID: &scope.folder2.Id})
-	want = int64(2)
-	if have != want {
+	// unread, newest first
+	unread := UNREAD
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false))
+	want = []string{"item011", "item121", "item111"}
+	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.Fail()
 	}
 
-	// feeds
-
-	have = db.CountItems(ItemFilter{FeedID: &scope.feed21.Id})
-	want = int64(2)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
-	have = db.CountItems(ItemFilter{FeedID: &scope.feed01.Id})
-	want = int64(3)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
-	// statuses
-
-	var unread ItemStatus = UNREAD
-	have = db.CountItems(ItemFilter{Status: &unread})
-	want = int64(3)
-	if have != want {
-		t.Logf("want: %#v", want)
-		t.Logf("have: %#v", have)
-		t.Fail()
-	}
-
-	// search
-
-	db.SyncSearch()
-	search := "title0"
-	have = db.CountItems(ItemFilter{Search: &search})
-	want = int64(3)
-	if have != want {
+	// starred, oldest first
+	starred := STARRED
+	have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false))
+	want = []string{"item212", "item013"}
+	if !reflect.DeepEqual(have, want) {
 		t.Logf("want: %#v", want)
 		t.Logf("have: %#v", have)
 		t.Fail()
@@ -256,7 +233,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db1 := testDB()
 	testItemsSetup(db1)
 	db1.MarkItemsRead(MarkFilter{})
-	have := testItemGuids(db1.ListItems(ItemFilter{Status: &read}, 0, 10, false))
+	have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want := []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item011", "item012",
@@ -270,7 +247,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db2 := testDB()
 	scope2 := testItemsSetup(db2)
 	db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
-	have = testItemGuids(db2.ListItems(ItemFilter{Status: &read}, 0, 10, false))
+	have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item121", "item122",
 		"item211", "item012",
@@ -284,7 +261,7 @@ func TestMarkItemsRead(t *testing.T) {
 	db3 := testDB()
 	scope3 := testItemsSetup(db3)
 	db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
-	have = testItemGuids(db3.ListItems(ItemFilter{Status: &read}, 0, 10, false))
+	have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
 	want = []string{
 		"item111", "item112", "item122",
 		"item211", "item012",
@@ -295,3 +272,59 @@ func TestMarkItemsRead(t *testing.T) {
 		t.Fail()
 	}
 }
+
+func TestDeleteOldItems(t *testing.T) {
+	extraItems := 10
+
+	now := time.Now().UTC()
+	db := testDB()
+	feed := db.CreateFeed("feed", "", "", "http://test.com/feed11.xml", nil)
+
+	items := make([]Item, 0)
+	for i := 0; i < itemsKeepSize+extraItems; i++ {
+		istr := strconv.Itoa(i)
+		items = append(items, Item{
+			GUID:   istr,
+			FeedId: feed.Id,
+			Title:  istr,
+			Date:   now.Add(time.Hour * time.Duration(i)),
+		})
+	}
+	db.CreateItems(items)
+
+	db.SetFeedSize(feed.Id, itemsKeepSize)
+	var feedSize int
+	err := db.db.QueryRow(
+		`select size from feed_sizes where feed_id = ?`, feed.Id,
+	).Scan(&feedSize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if feedSize != itemsKeepSize {
+		t.Fatalf(
+			"expected feed size to get updated\nwant: %d\nhave: %d",
+			itemsKeepSize+extraItems,
+			feedSize,
+		)
+	}
+
+	// expire only the first 3 articles
+	_, err = db.db.Exec(
+		`update items set date_arrived = ?
+		where id in (select id from items limit 3)`,
+		now.Add(-time.Hour*time.Duration(itemsKeepDays*24)),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db.DeleteOldItems()
+	feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false, false)
+	if len(feedItems) != len(items)-3 {
+		t.Fatalf(
+			"invalid number of old items kept\nwant: %d\nhave: %d",
+			len(items)-3,
+			len(feedItems),
+		)
+	}
+}

src/storage/migration.go

@@ -4,6 +4,7 @@ import (
 	"database/sql"
 	"fmt"
 	"log"
+	"time"
 )
 
 var migrations = []func(*sql.Tx) error{
@@ -13,13 +14,19 @@ var migrations = []func(*sql.Tx) error{
 	m04_item_podcasturl,
 	m05_move_description_to_content,
 	m06_fill_missing_dates,
+	m07_add_feed_size,
+	m08_normalize_datetime,
+	m09_change_item_index,
+	m10_add_item_medialinks,
 }
 
 var maxVersion = int64(len(migrations))
 
 func migrate(db *sql.DB) error {
 	var version int64
-	db.QueryRow("pragma user_version").Scan(&version)
+	if err := db.QueryRow("pragma user_version").Scan(&version); err != nil {
+		return err
+	}
 
 	if version >= maxVersion {
 		return nil
@@ -259,3 +266,69 @@ func m06_fill_missing_dates(tx *sql.Tx) error {
 	_, err := tx.Exec(sql)
 	return err
 }
+
+func m07_add_feed_size(tx *sql.Tx) error {
+	sql := `
+		create table if not exists feed_sizes (
+		 feed_id        references feeds(id) on delete cascade unique,
+		 size           integer not null default 0
+		);
+	`
+	_, err := tx.Exec(sql)
+	return err
+}
+
+func m08_normalize_datetime(tx *sql.Tx) error {
+	rows, err := tx.Query(`select id, date_arrived from items;`)
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var id int64
+		var dateArrived time.Time
+		err = rows.Scan(&id, &dateArrived)
+		if err != nil {
+			return err
+		}
+		_, err = tx.Exec(`update items set date_arrived = ? where id = ?;`, dateArrived.UTC(), id)
+		if err != nil {
+			return err
+		}
+	}
+	_, err = tx.Exec(`update items set date = strftime('%Y-%m-%d %H:%M:%f', date);`)
+	return err
+}
+
+func m09_change_item_index(tx *sql.Tx) error {
+	sql := `
+        drop index if exists idx_item_status;
+		create index if not exists idx_item__date_id_status on items(date,id,status);
+	`
+	_, err := tx.Exec(sql)
+	return err
+}
+
+func m10_add_item_medialinks(tx *sql.Tx) error {
+	sql := `
+		alter table items add column media_links blob;
+		update items set media_links =
+			iif(
+				coalesce(image, '') != '' and coalesce(podcast_url, '') != '',
+				json_array(json_object('type', 'image', 'url', image), json_object('type', 'audio', 'url', podcast_url)),
+				iif(
+					coalesce(image, '') != '',
+					json_array(json_object('type', 'image', 'url', image)),
+					iif(
+						coalesce(podcast_url, '') != '',
+						json_array(json_object('type', 'audio', 'url', podcast_url)),
+						null
+					)
+				)
+			);
+
+		alter table items drop column image;
+		alter table items drop column podcast_url;
+	`
+	_, err := tx.Exec(sql)
+	return err
+}

src/storage/storage.go

@@ -2,6 +2,9 @@ package storage
 
 import (
 	"database/sql"
+	"log"
+	"strings"
+
 	_ "github.com/mattn/go-sqlite3"
 )
 
@@ -10,13 +13,17 @@ type Storage struct {
 }
 
 func New(path string) (*Storage, error) {
+	if pos := strings.IndexRune(path, '?'); pos == -1 {
+		params := "_journal=WAL&_sync=NORMAL&_busy_timeout=5000&cache=shared"
+		log.Printf("opening db with params: %s", params)
+		path = path + "?" + params
+	}
+
 	db, err := sql.Open("sqlite3", path)
 	if err != nil {
 		return nil, err
 	}
 
-	db.SetMaxOpenConns(1)
-
 	if err = migrate(db); err != nil {
 		return nil, err
 	}

src/storage/storage_test.go

@@ -11,6 +11,7 @@ func testDB() *Storage {
 	log.SetOutput(io.Discard)
 	db, _ := New(":memory:")
 	log.SetOutput(os.Stderr)
+	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
 	return db
 }
 

src/systray/NOTES.txt

@@ -9,3 +9,4 @@ hash:
 changes:
 
 -removed `getlantern/golog` dependency
+-prevent from compiling in linux

src/systray/systray.go

@@ -1,3 +1,6 @@
+//go:build darwin || windows
+// +build darwin windows
+
 /*
 Package systray is a cross-platform Go library to place an icon and menu in the notification area.
 */