Compare commits

..

16 Commits

Author SHA1 Message Date
Nazar Kanaev
95aa3313bf [wip] 2024-12-04 22:02:35 +00:00
Nazar Kanaev
5254df53dc update docs 2024-12-04 21:44:10 +00:00
tillcash
7301eab99c Add Dynamic Disable Functionality to Navigation Buttons 2024-12-04 21:42:36 +00:00
Nazar Kanaev
ad138c3017 show article content in the list if the title is missing 2024-12-04 21:36:44 +00:00
Nazar Kanaev
b09c95d7ea navigation buttons (+ fix keyboard navigation in 1-pane view) 2024-12-04 15:57:34 +00:00
Nazar Kanaev
64611a0dd3 update changelog 2024-12-04 14:28:10 +00:00
Donovan Glover
321ad7608f fix: use noreferrer for external links
Based on the existing noreferrer code in the code base.
2024-11-28 12:58:00 +00:00
Nazar Kanaev
2a8b6ea935 update changelog 2024-11-25 21:50:38 +00:00
Nazar Kanaev
e9cbea500b autogenerate feed item guids 2024-11-25 21:35:04 +00:00
Nazar Kanaev
223039b2c6 do not send referrer for external images 2024-11-25 21:01:45 +00:00
funkycode
7402dfc4e6 fix the build 2024-11-09 23:10:43 +00:00
Tim Shaffer
6b12715506 Only read config dir if db is not provided. 2024-11-01 10:27:59 +00:00
Aidan Holm
2dc58c5c8e Fix user agent using hardcoded 1.0 version. 2024-10-30 09:49:33 +00:00
Nazar Kanaev
0cef51c6ac add sample links 2024-10-07 22:22:56 +01:00
Nazar Kanaev
2a4d974965 go fmt 2024-10-07 12:20:45 +01:00
dependabot[bot]
f71792d6a5 Bump actions/download-artifact from 2 to 4.1.7 in /.github/workflows
Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 2 to 4.1.7.
- [Release notes](https://github.com/actions/download-artifact/releases)
- [Commits](https://github.com/actions/download-artifact/compare/v2...v4.1.7)

---
updated-dependencies:
- dependency-name: actions/download-artifact
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-29 21:36:53 +01:00
20 changed files with 319 additions and 111 deletions

View File

@ -103,7 +103,7 @@ jobs:
draft: true
prerelease: true
- name: Download Artifacts
uses: actions/download-artifact@v2
uses: actions/download-artifact@v4.1.7
with:
path: .
- name: Preparation

View File

@ -13,6 +13,7 @@ import (
"github.com/nkanaev/yarr/src/platform"
"github.com/nkanaev/yarr/src/server"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/worker"
)
var Version string = "0.0"
@ -89,12 +90,12 @@ func main() {
log.SetOutput(os.Stdout)
}
configPath, err := os.UserConfigDir()
if err != nil {
log.Fatal("Failed to get config dir: ", err)
}
if db == "" {
configPath, err := os.UserConfigDir()
if err != nil {
log.Fatal("Failed to get config dir: ", err)
}
storagePath := filepath.Join(configPath, "yarr")
if err := os.MkdirAll(storagePath, 0755); err != nil {
log.Fatal("Failed to create app config dir: ", err)
@ -105,6 +106,7 @@ func main() {
log.Printf("using db file %s", db)
var username, password string
var err error
if authfile != "" {
f, err := os.Open(authfile)
if err != nil {
@ -131,6 +133,7 @@ func main() {
log.Fatal("Failed to initialise database: ", err)
}
worker.SetVersion(Version)
srv := server.NewServer(store, addr)
if basepath != "" {

View File

@ -3,7 +3,7 @@
- (new) Fever API support (thanks to @icefed)
- (new) editable feed link (thanks to @adaszko)
- (new) switch to feed by clicking the title in the article page (thanks to @tarasglek for suggestion)
- (new) parser rewritten to support multiple media links
- (new) support multiple media links
- (fix) duplicate articles caused by the same feed addition (thanks to @adaszko)
- (fix) relative article links (thanks to @adazsko for the report)
- (fix) atom article links stored in id element (thanks to @adazsko for the report)
@ -11,6 +11,12 @@
- (fix) sorting same-day batch articles (thanks to @lamescholar for the report)
- (fix) showing login page in the selected theme (thanks to @feddiriko for the report)
- (fix) parsing atom feeds with html elements (thanks to @tillcash & @toBeOfUse for the report, @krkk for the fix)
- (fix) parsing feeds with missing guids (thanks to @hoyii for the report)
- (fix) sending actual client version to servers (thanks to @aidanholm)
- (fix) error caused by missing config dir (thanks to @timster)
- (etc) load external images with no-referrer policy (thanks to @tillcash for the report)
- (etc) open external links with no-referrer policy (thanks to @donovanglover)
- (etc) show article content in the list if title is missing (thanks to @asimpson for suggestion)
# v2.4 (2023-08-15)

68
doc/samples.yml Normal file
View File

@ -0,0 +1,68 @@
- site: https://vimeo.com/channels/staffpicks/videos
feed: https://vimeo.com/channels/staffpicks/videos/rss
tags: [vimeo, image]
- site: https://www.youtube.com/@everyframeapainting/videos
feed: https://www.youtube.com/feeds/videos.xml?channel_id=UCjFqcJQXGZ6T6sxyFB-5i6A"
tags: [youtube, image]
- site: https://iwdrm.tumblr.com/
feed: https://iwdrm.tumblr.com/rss
tags: [tumblr, image]
- site: https://falseknees.tumblr.com/
feed: https://falseknees.tumblr.com/rss
tags: [tumblr, image]
- site: https://accidentallyquadratic.tumblr.com/
feed: https://accidentallyquadratic.tumblr.com/rss
info: text blog with code sections
tags: [tumblr, text, code]
- site: https://www.flickr.com/photos/maratsafin/
feed: https://www.flickr.com/services/feeds/photos_public.gne?id=59021497@N07&lang=en-us&format=atom
tags: [flickr, image]
- site: https://www.reddit.com/r/comics
feed: https://www.reddit.com/r/comics.rss
tags: [reddit, image]
- site: https://www.reddit.com/r/AITAH
feed: https://www.reddit.com/r/AITAH.rss
tags: [reddit, text]
- site: https://idothei.wordpress.com/
feed: https://idothei.wordpress.com/feed/
tags: [wordpress, text]
- site: https://www.vidarholen.net/contents/blog/
feed: https://www.vidarholen.net/contents/blog/?feed=rss2
tags: [wordpress, text]
- site: https://blog.posthaven.com/
feed: https://blog.posthaven.com/posts.atom
tags: [posthaven, text]
- site: https://medium.com/@dailynewsletter
feed: https://medium.com/feed/@dailynewsletter
tags: [medium, text]
- site: https://thereveal.substack.com/
feed: https://thereveal.substack.com/feed
tags: [substack, text]
- site: https://tema.livejournal.com/
feed: https://tema.livejournal.com/data/rss
tags: [livejournal, text]
- site: https://mametter.hatenablog.com/
feed: https://mametter.hatenablog.com/feed
tags: [hatena, text]
- site: https://juliepowell.blogspot.com/
feed: https://juliepowell.blogspot.com/feeds/posts/default
tags: [blogger, text]
- site: https://micro.blog/val
feed: https://micro.blog/posts/val
tags: [json, microblog]

View File

@ -207,11 +207,11 @@
<span class="icon">{% inline "more-horizontal.svg" %}</span>
</template>
<header class="dropdown-header">{{ current.feed.title }}</header>
<a class="dropdown-item" :href="current.feed.link" target="_blank" v-if="current.feed.link">
<a class="dropdown-item" :href="current.feed.link" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer" v-if="current.feed.link">
<span class="icon mr-1">{% inline "globe.svg" %}</span>
Website
</a>
<a class="dropdown-item" :href="current.feed.feed_link" target="_blank" v-if="current.feed.feed_link">
<a class="dropdown-item" :href="current.feed.feed_link" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer" v-if="current.feed.feed_link">
<span class="icon mr-1">{% inline "rss.svg" %}</span>
Feed Link
</a>
@ -326,10 +326,16 @@
title="Read Here">
<span class="icon" :class="{'icon-loading': loading.readability}">{% inline "book-open.svg" %}</span>
</button>
<a class="toolbar-item" :href="itemSelectedDetails.link" target="_blank" title="Open Link">
<a class="toolbar-item" :href="itemSelectedDetails.link" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer" title="Open Link">
<span class="icon">{% inline "external-link.svg" %}</span>
</a>
<div class="flex-grow-1"></div>
<button class="toolbar-item" @click="navigateToItem(-1)" title="Previous Article" :disabled="itemSelected == items[0].id">
<span class="icon">{% inline "chevron-left.svg" %}</span>
</button>
<button class="toolbar-item" @click="navigateToItem(+1)" title="Next Article" :disabled="itemSelected == items[items.length - 1].id">
<span class="icon">{% inline "chevron-right.svg" %}</span>
</button>
<button class="toolbar-item" @click="itemSelected=null" title="Close Article">
<span class="icon">{% inline "x.svg" %}</span>
</button>

View File

@ -2,6 +2,26 @@
var TITLE = document.title
function scrollto(target, scroll) {
var padding = 10
var targetRect = target.getBoundingClientRect()
var scrollRect = scroll.getBoundingClientRect()
// target
var relativeOffset = targetRect.y - scrollRect.y
var absoluteOffset = relativeOffset + scroll.scrollTop
if (padding <= relativeOffset && relativeOffset + targetRect.height <= scrollRect.height - padding) return
var newPos = scroll.scrollTop
if (relativeOffset < padding) {
newPos = absoluteOffset - padding
} else {
newPos = absoluteOffset - scrollRect.height + targetRect.height + padding
}
scroll.scrollTop = Math.round(newPos)
}
var debounce = function(callback, wait) {
var timeout
return function() {
@ -415,7 +435,7 @@ var vm = new Vue({
vm.feeds = values[1]
})
},
refreshItems: function(loadMore) {
refreshItems: function(loadMore = false) {
if (this.feedSelected === null) {
vm.items = []
vm.itemsHasMore = false
@ -428,7 +448,7 @@ var vm = new Vue({
}
this.loading.items = true
api.items.list(query).then(function(data) {
return api.items.list(query).then(function(data) {
if (loadMore) {
vm.items = vm.items.concat(data.list)
} else {
@ -451,13 +471,17 @@ var vm = new Vue({
var scale = (parseFloat(getComputedStyle(document.documentElement).fontSize) || 16) / 16
var el = this.$refs.itemlist
if (el.scrollHeight === 0) return false // element is invisible (responsive design)
var closeToBottom = (el.scrollHeight - el.scrollTop - el.offsetHeight) < bottomSpace * scale
return closeToBottom
},
loadMoreItems: function(event, el) {
if (!this.itemsHasMore) return
if (this.loading.items) return
if (this.itemListCloseToBottom()) this.refreshItems(true)
if (this.itemListCloseToBottom()) return this.refreshItems(true)
if (this.itemSelected && this.itemSelected === this.items[this.items.length - 1].id) return this.refreshItems(true)
},
markItemsRead: function() {
var query = this.getItemsQuery()
@ -691,6 +715,65 @@ var vm = new Vue({
this.filteredFolderStats = statsFolders
this.filteredTotalStats = statsTotal
},
// navigation helper, navigate relative to selected item
navigateToItem: function(relativePosition) {
let vm = this
if (vm.itemSelected == null) {
// if no item is selected, select first
if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
return
}
var itemPosition = vm.items.findIndex(function(x) { return x.id === vm.itemSelected })
if (itemPosition === -1) {
if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
return
}
var newPosition = itemPosition + relativePosition
if (newPosition < 0 || newPosition >= vm.items.length) return
vm.itemSelected = vm.items[newPosition].id
vm.$nextTick(function() {
var scroll = document.querySelector('#item-list-scroll')
var handle = scroll.querySelector('input[type=radio]:checked')
var target = handle && handle.parentElement
if (target && scroll) scrollto(target, scroll)
vm.loadMoreItems()
})
},
// navigation helper, navigate relative to selected feed
navigateToFeed: function(relativePosition) {
let vm = this
var navigationList = Array.from(document.querySelectorAll('#col-feed-list input[name=feed]'))
.filter(function(r) { return r.offsetParent !== null && r.value !== 'folder:null' })
.map(function(r) { return r.value })
var currentFeedPosition = navigationList.indexOf(vm.feedSelected)
if (currentFeedPosition == -1) {
vm.feedSelected = ''
return
}
var newPosition = currentFeedPosition+relativePosition
if (newPosition < 0 || newPosition >= navigationList.length) return
vm.feedSelected = navigationList[newPosition]
vm.$nextTick(function() {
var scroll = document.querySelector('#feed-list-scroll')
var handle = scroll.querySelector('input[type=radio]:checked')
var target = handle && handle.parentElement
if (target && scroll) scrollto(target, scroll)
})
},
}
})

View File

@ -1,79 +1,4 @@
function scrollto(target, scroll) {
var padding = 10
var targetRect = target.getBoundingClientRect()
var scrollRect = scroll.getBoundingClientRect()
// target
var relativeOffset = targetRect.y - scrollRect.y
var absoluteOffset = relativeOffset + scroll.scrollTop
if (padding <= relativeOffset && relativeOffset + targetRect.height <= scrollRect.height - padding) return
var newPos = scroll.scrollTop
if (relativeOffset < padding) {
newPos = absoluteOffset - padding
} else {
newPos = absoluteOffset - scrollRect.height + targetRect.height + padding
}
scroll.scrollTop = Math.round(newPos)
}
var helperFunctions = {
// navigation helper, navigate relative to selected item
navigateToItem: function(relativePosition) {
if (vm.itemSelected == null) {
// if no item is selected, select first
if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
return
}
var itemPosition = vm.items.findIndex(function(x) { return x.id === vm.itemSelected })
if (itemPosition === -1) {
if (vm.items.length !== 0) vm.itemSelected = vm.items[0].id
return
}
var newPosition = itemPosition + relativePosition
if (newPosition < 0 || newPosition >= vm.items.length) return
vm.itemSelected = vm.items[newPosition].id
vm.$nextTick(function() {
var scroll = document.querySelector('#item-list-scroll')
var handle = scroll.querySelector('input[type=radio]:checked')
var target = handle && handle.parentElement
if (target && scroll) scrollto(target, scroll)
})
},
// navigation helper, navigate relative to selected feed
navigateToFeed: function(relativePosition) {
var navigationList = Array.from(document.querySelectorAll('#col-feed-list input[name=feed]'))
.filter(function(r) { return r.offsetParent !== null && r.value !== 'folder:null' })
.map(function(r) { return r.value })
var currentFeedPosition = navigationList.indexOf(vm.feedSelected)
if (currentFeedPosition == -1) {
vm.feedSelected = ''
return
}
var newPosition = currentFeedPosition+relativePosition
if (newPosition < 0 || newPosition >= navigationList.length) return
vm.feedSelected = navigationList[newPosition]
vm.$nextTick(function() {
var scroll = document.querySelector('#feed-list-scroll')
var handle = scroll.querySelector('input[type=radio]:checked')
var target = handle && handle.parentElement
if (target && scroll) scrollto(target, scroll)
})
},
scrollContent: function(direction) {
var padding = 40
var scroll = document.querySelector('.content')
@ -92,7 +17,7 @@ var helperFunctions = {
var shortcutFunctions = {
openItemLink: function() {
if (vm.itemSelectedDetails && vm.itemSelectedDetails.link) {
window.open(vm.itemSelectedDetails.link, '_blank')
window.open(vm.itemSelectedDetails.link, '_blank', 'noopener,noreferrer')
}
},
toggleReadability: function() {
@ -118,16 +43,16 @@ var shortcutFunctions = {
document.getElementById("searchbar").focus()
},
nextItem(){
helperFunctions.navigateToItem(+1)
vm.navigateToItem(+1)
},
previousItem() {
helperFunctions.navigateToItem(-1)
vm.navigateToItem(-1)
},
nextFeed(){
helperFunctions.navigateToFeed(+1)
vm.navigateToFeed(+1)
},
previousFeed() {
helperFunctions.navigateToFeed(-1)
vm.navigateToFeed(-1)
},
scrollForward: function() {
helperFunctions.scrollContent(+1)

View File

@ -4,6 +4,7 @@ import (
"bytes"
"regexp"
"strings"
"unicode"
"golang.org/x/net/html"
)
@ -61,3 +62,16 @@ func ExtractText(content string) string {
text = whitespaceRegex.ReplaceAllLiteralString(text, " ")
return text
}
func TruncateText(input string, size int) string {
runes := []rune(input)
if len(runes) <= size {
return input
}
for i := size - 1; i > 0; i-- {
if unicode.IsSpace(runes[i]) {
return string(runes[:i]) + " ..."
}
}
return input
}

View File

@ -24,3 +24,21 @@ func TestExtractText(t *testing.T) {
}
}
}
func TestTruncateText(t *testing.T) {
input := "Lorem ipsum — классический текст-«рыба»"
size := 30
want := "Lorem ipsum — классический ..."
have := TruncateText(input, size)
if want != have {
t.Errorf("\nsize: %d\nwant: %#v\nhave: %#v", size, want, have)
}
size = 1000
want = input
have = TruncateText(input, size)
if want != have {
t.Errorf("\nsize: %d\nwant: %#v\nhave: %#v", size, want, have)
}
}

View File

@ -167,7 +167,7 @@ func getExtraAttributes(tagName string) ([]string, []string) {
case "iframe":
return []string{"sandbox", "loading"}, []string{`sandbox="allow-scripts allow-same-origin allow-popups"`, `loading="lazy"`}
case "img":
return []string{"loading"}, []string{`loading="lazy"`}
return []string{"loading"}, []string{`loading="lazy"`, `referrerpolicy="no-referrer"`}
default:
return nil, nil
}

View File

@ -2,6 +2,7 @@ package parser
import (
"bytes"
"crypto/sha256"
"encoding/xml"
"errors"
"fmt"
@ -119,6 +120,7 @@ func ParseAndFix(r io.Reader, baseURL, fallbackEncoding string) (*Feed, error) {
}
feed.TranslateURLs(baseURL)
feed.SetMissingDatesTo(time.Now())
feed.SetMissingGUIDs()
return feed, nil
}
@ -171,3 +173,12 @@ func (feed *Feed) TranslateURLs(base string) error {
}
return nil
}
func (feed *Feed) SetMissingGUIDs() {
for i, item := range feed.Items {
if item.GUID == "" {
id := strings.Join([]string{item.Title, item.Date.Format(time.RFC3339), item.URL}, ";;")
feed.Items[i].GUID = fmt.Sprintf("%x", sha256.Sum256([]byte(id)))
}
}
}

View File

@ -150,3 +150,32 @@ func TestParseCleanIllegalCharsInNonUTF8(t *testing.T) {
t.Fatalf("invalid feed, got: %v", feed)
}
}
func TestParseMissingGUID(t *testing.T) {
data := `
<?xml version="1.0" encoding="windows-1251"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<item>
<title>foo</title>
</item>
<item>
<title>bar</title>
</item>
</channel>
</rss>
`
feed, err := ParseAndFix(strings.NewReader(data), "", "")
if err != nil {
t.Fatal(err)
}
if len(feed.Items) != 2 {
t.Fatalf("expected 2 items, got %d", len(feed.Items))
}
if feed.Items[0].GUID == "" || feed.Items[1].GUID == "" {
t.Fatalf("item GUIDs are missing, got %#v", feed.Items)
}
if feed.Items[0].GUID == feed.Items[1].GUID {
t.Fatalf("item GUIDs are not unique, got %#v", feed.Items)
}
}

View File

@ -71,10 +71,10 @@ func (m *media) mediaLinks() []MediaLink {
for _, group := range m.MediaGroups {
for _, thumbnail := range group.MediaThumbnails {
links = append(links, MediaLink{
URL: thumbnail.URL,
Type: "image",
URL: thumbnail.URL,
Type: "image",
})
}
}
}
for _, content := range m.MediaContents {
if content.MediaURL != "" {
@ -107,7 +107,7 @@ func (m *media) mediaLinks() []MediaLink {
}
for _, thumbnail := range content.MediaThumbnails {
links = append(links, MediaLink{
URL: thumbnail.URL,
URL: thumbnail.URL,
Type: "image",
})
}

View File

@ -19,7 +19,7 @@ type Item struct {
}
type MediaLink struct {
URL string
Type string
URL string
Type string
Description string
}

View File

@ -62,7 +62,6 @@ type rssEnclosure struct {
func ParseRSS(r io.Reader) (*Feed, error) {
srcfeed := rssFeed{}
// comment
decoder := xmlDecoder(r)
decoder.DefaultSpace = "rss"

View File

@ -247,3 +247,38 @@ func TestRSSIsPermalink(t *testing.T) {
}
}
}
func TestRSSMultipleMedia(t *testing.T) {
feed, _ := Parse(strings.NewReader(`
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
<channel>
<item>
<guid isPermaLink="true">http://example.com/posts/1</guid>
<media:content url="https://example.com/path/to/image1.png" type="image/png" fileSize="1000" medium="image">
<media:description type="plain">description 1</media:description>
</media:content>
<media:content url="https://example.com/path/to/image2.png" type="image/png" fileSize="2000" medium="image">
<media:description type="plain">description 2</media:description>
</media:content>
</item>
</channel>
</rss>
`))
have := feed.Items
want := []Item{
{
GUID: "http://example.com/posts/1",
URL: "http://example.com/posts/1",
MediaLinks: []MediaLink{
{URL:"https://example.com/path/to/image1.png", Type:"image", Description:"description 1"},
{URL:"https://example.com/path/to/image2.png", Type:"image", Description:"description 2"},
},
},
}
if !reflect.DeepEqual(want, have) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fatal("invalid rss")
}
}

View File

@ -374,12 +374,19 @@ func (s *Server) handleItemList(c *router.Context) {
}
newestFirst := query.Get("oldest_first") != "true"
items := s.db.ListItems(filter, perPage+1, newestFirst, false)
items := s.db.ListItems(filter, perPage+1, newestFirst, true)
hasMore := false
if len(items) == perPage+1 {
hasMore = true
items = items[:perPage]
}
for i, item := range items {
if item.Title == "" {
text := htmlutil.ExtractText(item.Content)
items[i].Title = htmlutil.TruncateText(text, 140)
}
}
c.JSON(http.StatusOK, map[string]interface{}{
"list": items,
"has_more": hasMore,

View File

@ -46,8 +46,8 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error {
}
type MediaLink struct {
URL string `json:"url"`
Type string `json:"type"`
URL string `json:"url"`
Type string `json:"type"`
Description string `json:"description,omitempty"`
}

View File

@ -32,6 +32,10 @@ func (c *Client) getConditional(url, lastModified, etag string) (*http.Response,
var client *Client
func SetVersion(num string) {
client.userAgent = "Yarr/" + num
}
func init() {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,

View File

@ -148,13 +148,13 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
mediaLinks = append(mediaLinks, storage.MediaLink(link))
}
result[i] = storage.Item{
GUID: item.GUID,
FeedId: feed.Id,
Title: item.Title,
Link: item.URL,
Content: item.Content,
Date: item.Date,
Status: storage.UNREAD,
GUID: item.GUID,
FeedId: feed.Id,
Title: item.Title,
Link: item.URL,
Content: item.Content,
Date: item.Date,
Status: storage.UNREAD,
MediaLinks: mediaLinks,
}
}