james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

support multiple media links

Browse Source
This commit is contained in:
Nazar Kanaev 2024-09-25 12:32:23 +01:00 committed by nkanaev
parent 097a2da5cb
commit b8afa82a81
17 changed files with 291 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/changelog.txt
View File

@ -3,6 +3,7 @@
- (new) Fever API support (thanks to @icefed)
- (new) editable feed link (thanks to @adaszko)
- (new) switch to feed by clicking the title in the article page (thanks to @tarasglek for suggestion)
- (new) support multiple media links
- (fix) duplicate articles caused by the same feed addition (thanks to @adaszko)
- (fix) relative article links (thanks to @adazsko for the report)
- (fix) atom article links stored in id element (thanks to @adazsko for the report)

2
go.mod
View File

@ -1,6 +1,6 @@
module github.com/nkanaev/yarr
go 1.17
go 1.18
require (
github.com/mattn/go-sqlite3 v1.14.7

13
makefile
View File

@ -1,6 +1,7 @@
VERSION=2.4
GITHASH=$(shell git rev-parse --short=8 HEAD)
GO_TAGS = sqlite_foreign_keys sqlite_json
GO_LDFLAGS = -s -w -X 'main.Version=$(VERSION)' -X 'main.GitHash=$(GITHASH)'
export GOARCH ?= amd64
@ -8,26 +9,26 @@ export CGO_ENABLED = 1
build_default:
mkdir -p _output
go build -tags "sqlite_foreign_keys" -ldflags="$(GO_LDFLAGS)" -o _output/yarr ./cmd/yarr
go build -tags "$(GO_TAGS)" -ldflags="$(GO_LDFLAGS)" -o _output/yarr ./cmd/yarr
build_macos:
mkdir -p _output/macos
GOOS=darwin go build -tags "sqlite_foreign_keys macos" -ldflags="$(GO_LDFLAGS)" -o _output/macos/yarr ./cmd/yarr
GOOS=darwin go build -tags "$(GO_TAGS) macos" -ldflags="$(GO_LDFLAGS)" -o _output/macos/yarr ./cmd/yarr
cp src/platform/icon.png _output/macos/icon.png
go run ./cmd/package_macos -outdir _output/macos -version "$(VERSION)"
build_linux:
mkdir -p _output/linux
GOOS=linux go build -tags "sqlite_foreign_keys linux" -ldflags="$(GO_LDFLAGS)" -o _output/linux/yarr ./cmd/yarr
GOOS=linux go build -tags "$(GO_TAGS) linux" -ldflags="$(GO_LDFLAGS)" -o _output/linux/yarr ./cmd/yarr
build_windows:
mkdir -p _output/windows
go run ./cmd/generate_versioninfo -version "$(VERSION)" -outfile src/platform/versioninfo.rc
windres -i src/platform/versioninfo.rc -O coff -o src/platform/versioninfo.syso
GOOS=windows go build -tags "sqlite_foreign_keys windows" -ldflags="$(GO_LDFLAGS) -H windowsgui" -o _output/windows/yarr.exe ./cmd/yarr
GOOS=windows go build -tags "$(GO_TAGS) windows" -ldflags="$(GO_LDFLAGS) -H windowsgui" -o _output/windows/yarr.exe ./cmd/yarr
serve:
go run -tags "sqlite_foreign_keys" ./cmd/yarr -db local.db
go run -tags "$(GO_TAGS)" ./cmd/yarr -db local.db
test:
go test -tags "sqlite_foreign_keys" ./...
go test -tags "$(GO_TAGS)" ./...

10
src/assets/index.html
View File

@ -362,8 +362,14 @@
</div>
<hr>
<div v-if="!itemSelectedReadability">
<img :src="itemSelectedDetails.image" v-if="itemSelectedDetails.image" class="mb-3">
<audio class="w-100" controls v-if="itemSelectedDetails.podcast_url" :src="itemSelectedDetails.podcast_url"></audio>
<div v-if="contentImages.length">
<figure v-for="media in contentImages">
<img :src="media.url" loading="lazy">
<figcaption v-if="media.description" v-html="media.description"></figcaption>
</figure>
</div>
<audio class="w-100" controls v-for="media in contentAudios" :src="media.url"></audio>
<video class="w-100" controls v-for="media in contentVideos" :src="media.url"></video>
</div>
<div v-html="itemSelectedContent"></div>
</div>

12
src/assets/javascripts/app.js
View File

@ -298,6 +298,18 @@ var vm = new Vue({
return this.itemSelectedDetails.content || ''
},
contentImages: function() {
if (!this.itemSelectedDetails) return []
return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'image')
},
contentAudios: function() {
if (!this.itemSelectedDetails) return []
return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'audio')
},
contentVideos: function() {
if (!this.itemSelectedDetails) return []
return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'video')
}
},
watch: {
'theme': {

5
src/parser/atom.go
View File

@ -89,6 +89,8 @@ func ParseAtom(r io.Reader) (*Feed, error) {
guidFromID = srcitem.ID + "::" + srcitem.Updated
}
mediaLinks := srcitem.mediaLinks()
link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""), linkFromID)
dstfeed.Items = append(dstfeed.Items, Item{
GUID: firstNonEmpty(guidFromID, srcitem.ID, link),
@ -96,8 +98,7 @@ func ParseAtom(r io.Reader) (*Feed, error) {
URL: link,
Title: srcitem.Title.Text(),
Content: firstNonEmpty(srcitem.Content.String(), srcitem.Summary.String(), srcitem.firstMediaDescription()),
ImageURL: srcitem.firstMediaThumbnail(),
AudioURL: "",
MediaLinks: mediaLinks,
})
}
return dstfeed, nil

18
src/parser/atom_test.go
View File

@ -45,8 +45,6 @@ func TestAtom(t *testing.T) {
URL: "http://example.org/2003/12/13/atom03.html",
Title: "Atom-Powered Robots Run Amok",
Content: `<div xmlns="http://www.w3.org/1999/xhtml"><p>This is the entry content.</p></div>`,
ImageURL: "",
AudioURL: "",
},
},
}
@ -141,9 +139,15 @@ func TestAtomImageLink(t *testing.T) {
</entry>
</feed>
`))
have := feed.Items[0].ImageURL
want := `https://example.com/image.png?width=100&height=100`
if want != have {
if len(feed.Items[0].MediaLinks) != 1 {
t.Fatalf("Expected 1 media link, got: %#v", feed.Items[0].MediaLinks)
}
have := feed.Items[0].MediaLinks[0]
want := MediaLink{
URL: `https://example.com/image.png?width=100&height=100`,
Type: "image",
}
if !reflect.DeepEqual(want, have) {
t.Fatalf("item.image_url doesn't match\nwant: %#v\nhave: %#v\n", want, have)
}
}
@ -165,8 +169,8 @@ func TestAtomImageLinkDuplicated(t *testing.T) {
if want != have {
t.Fatalf("want: %#v\nhave: %#v\n", want, have)
}
if feed.Items[0].ImageURL != "" {
t.Fatal("item.image_url must be unset if present in the content")
if len(feed.Items[0].MediaLinks) != 0 {
t.Fatal("item media link must be excluded if present in the content")
}
}

11
src/parser/feed.go
View File

@ -134,11 +134,14 @@ func (feed *Feed) cleanup() {
feed.Items[i].Title = strings.TrimSpace(htmlutil.ExtractText(item.Title))
feed.Items[i].Content = strings.TrimSpace(item.Content)
if item.ImageURL != "" && strings.Contains(item.Content, item.ImageURL) {
feed.Items[i].ImageURL = ""
if len(feed.Items[i].MediaLinks) > 0 {
mediaLinks := make([]MediaLink, 0)
for _, link := range item.MediaLinks {
if !strings.Contains(item.Content, link.URL) {
mediaLinks = append(mediaLinks, link)
}
if item.AudioURL != "" && strings.Contains(item.Content, item.AudioURL) {
feed.Items[i].AudioURL = ""
}
feed.Items[i].MediaLinks = mediaLinks
}
}
}

62
src/parser/media.go
View File

@ -1,5 +1,9 @@
package parser
import (
"strings"
)
type media struct {
MediaGroups []mediaGroup `xml:"http://search.yahoo.com/mrss/ group"`
MediaContents []mediaContent `xml:"http://search.yahoo.com/mrss/ content"`
@ -8,12 +12,17 @@ type media struct {
}
type mediaGroup struct {
MediaContent []mediaContent `xml:"http://search.yahoo.com/mrss/ content"`
MediaThumbnails []mediaThumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"`
MediaDescriptions []mediaDescription `xml:"http://search.yahoo.com/mrss/ description"`
}
type mediaContent struct {
MediaThumbnails []mediaThumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"`
MediaType string `xml:"type,attr"`
MediaMedium string `xml:"medium,attr"`
MediaURL string `xml:"url,attr"`
MediaDescription mediaDescription `xml:"http://search.yahoo.com/mrss/ description"`
}
type mediaThumbnail struct {
@ -22,7 +31,7 @@ type mediaThumbnail struct {
type mediaDescription struct {
Type string `xml:"type,attr"`
Description string `xml:",chardata"`
Text string `xml:",chardata"`
}
func (m *media) firstMediaThumbnail() string {
@ -44,12 +53,59 @@ func (m *media) firstMediaThumbnail() string {
func (m *media) firstMediaDescription() string {
for _, d := range m.MediaDescriptions {
return plain2html(d.Description)
return plain2html(d.Text)
}
for _, g := range m.MediaGroups {
for _, d := range g.MediaDescriptions {
return plain2html(d.Description)
return plain2html(d.Text)
}
}
return ""
}
func (m *media) mediaLinks() []MediaLink {
links := make([]MediaLink, 0)
for _, thumbnail := range m.MediaThumbnails {
links = append(links, MediaLink{URL: thumbnail.URL, Type: "image"})
}
for _, group := range m.MediaGroups {
for _, thumbnail := range group.MediaThumbnails {
links = append(links, MediaLink{
URL: thumbnail.URL,
Type: "image",
})
}
}
for _, content := range m.MediaContents {
if content.MediaURL != "" {
url := content.MediaURL
description := content.MediaDescription.Text
if strings.HasPrefix(content.MediaType, "image/") {
links = append(links, MediaLink{URL: url, Type: "image", Description: description})
} else if strings.HasPrefix(content.MediaType, "audio/") {
links = append(links, MediaLink{URL: url, Type: "audio", Description: description})
} else if strings.HasPrefix(content.MediaType, "video/") {
links = append(links, MediaLink{URL: url, Type: "video", Description: description})
} else if content.MediaMedium == "image" || content.MediaMedium == "audio" || content.MediaMedium == "video" {
links = append(links, MediaLink{URL: url, Type: content.MediaMedium, Description: description})
} else {
if len(content.MediaThumbnails) > 0 {
links = append(links, MediaLink{
URL: content.MediaThumbnails[0].URL,
Type: "image",
})
}
}
}
for _, thumbnail := range content.MediaThumbnails {
links = append(links, MediaLink{
URL: thumbnail.URL,
Type: "image",
})
}
}
if len(links) == 0 {
return nil
}
return links
}

9
src/parser/models.go
View File

@ -15,6 +15,11 @@ type Item struct {
Title string
Content string
ImageURL string
AudioURL string
MediaLinks []MediaLink
}
type MediaLink struct {
URL string
Type string
Description string
}

11
src/parser/rss.go
View File

@ -74,14 +74,14 @@ func ParseRSS(r io.Reader) (*Feed, error) {
SiteURL: srcfeed.Link,
}
for _, srcitem := range srcfeed.Items {
podcastURL := ""
mediaLinks := srcitem.mediaLinks()
for _, e := range srcitem.Enclosures {
if strings.HasPrefix(e.Type, "audio/") {
podcastURL = e.URL
podcastURL := e.URL
if srcitem.OrigEnclosureLink != "" && strings.Contains(podcastURL, path.Base(srcitem.OrigEnclosureLink)) {
podcastURL = srcitem.OrigEnclosureLink
}
mediaLinks = append(mediaLinks, MediaLink{URL: podcastURL, Type: "audio"})
break
}
}
@ -96,9 +96,8 @@ func ParseRSS(r io.Reader) (*Feed, error) {
Date: dateParse(firstNonEmpty(srcitem.DublinCoreDate, srcitem.PubDate)),
URL: firstNonEmpty(srcitem.OrigLink, srcitem.Link, permalink),
Title: srcitem.Title,
Content: firstNonEmpty(srcitem.ContentEncoded, srcitem.Description),
AudioURL: podcastURL,
ImageURL: srcitem.firstMediaThumbnail(),
Content: firstNonEmpty(srcitem.ContentEncoded, srcitem.Description, srcitem.firstMediaDescription()),
MediaLinks: mediaLinks,
})
}
return dstfeed, nil

82
src/parser/rss_test.go
View File

@ -75,9 +75,15 @@ func TestRSSMediaContentThumbnail(t *testing.T) {
</channel>
</rss>
`))
have := feed.Items[0].ImageURL
want := "https://i.vimeocdn.com/video/1092705247_960.jpg"
if have != want {
if len(feed.Items[0].MediaLinks) != 1 {
t.Fatalf("Expected 1 media link, got %#v", feed.Items[0].MediaLinks)
}
have := feed.Items[0].MediaLinks[0]
want := MediaLink{
URL: "https://i.vimeocdn.com/video/1092705247_960.jpg",
Type: "image",
}
if !reflect.DeepEqual(want, have) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.FailNow()
@ -127,9 +133,15 @@ func TestRSSPodcast(t *testing.T) {
</channel>
</rss>
`))
have := feed.Items[0].AudioURL
want := "http://example.com/audio.ext"
if want != have {
if len(feed.Items[0].MediaLinks) != 1 {
t.Fatal("Invalid media links")
}
have := feed.Items[0].MediaLinks[0]
want := MediaLink{
URL: "http://example.com/audio.ext",
Type: "audio",
}
if !reflect.DeepEqual(want, have) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.FailNow()
@ -147,9 +159,15 @@ func TestRSSOpusPodcast(t *testing.T) {
</channel>
</rss>
`))
have := feed.Items[0].AudioURL
want := "http://example.com/audio.ext"
if want != have {
if len(feed.Items[0].MediaLinks) != 1 {
t.Fatal("Invalid media links")
}
have := feed.Items[0].MediaLinks[0]
want := MediaLink{
URL: "http://example.com/audio.ext",
Type: "audio",
}
if !reflect.DeepEqual(want, have) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.FailNow()
@ -176,8 +194,9 @@ func TestRSSPodcastDuplicated(t *testing.T) {
if want != have {
t.Fatalf("content doesn't match\nwant: %#v\nhave: %#v\n", want, have)
}
if feed.Items[0].AudioURL != "" {
t.Fatal("item.audio_url must be unset if present in the content")
if len(feed.Items[0].MediaLinks) != 0 {
t.Fatal("item media must be excluded if present in the content")
}
}
@ -223,8 +242,47 @@ func TestRSSIsPermalink(t *testing.T) {
},
}
for i := 0; i < len(want); i++ {
if want[i] != have[i] {
if !reflect.DeepEqual(want, have) {
t.Errorf("Failed to handle isPermalink\nwant: %#v\nhave: %#v\n", want[i], have[i])
}
}
}
func TestRSSMultipleMedia(t *testing.T) {
feed, _ := Parse(strings.NewReader(`
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
<channel>
<item>
<guid isPermaLink="true">http://example.com/posts/1</guid>
<media:content url="https://example.com/path/to/image1.png" type="image/png" fileSize="1000" medium="image">
<media:description type="plain">description 1</media:description>
</media:content>
<media:content url="https://example.com/path/to/image2.png" type="image/png" fileSize="2000" medium="image">
<media:description type="plain">description 2</media:description>
</media:content>
<media:content url="https://example.com/path/to/video1.mp4" type="video/mp4" fileSize="2000" medium="image">
<media:description type="plain">video description</media:description>
</media:content>
</item>
</channel>
</rss>
`))
have := feed.Items
want := []Item{
{
GUID: "http://example.com/posts/1",
URL: "http://example.com/posts/1",
MediaLinks: []MediaLink{
{URL:"https://example.com/path/to/image1.png", Type:"image", Description:"description 1"},
{URL:"https://example.com/path/to/image2.png", Type:"image", Description:"description 2"},
{URL:"https://example.com/path/to/video1.mp4", Type:"video", Description:"video description"},
},
},
}
if !reflect.DeepEqual(want, have) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fatal("invalid rss")
}
}

3
src/server/routes.go
View File

@ -329,6 +329,9 @@ func (s *Server) handleItem(c *router.Context) {
}
item.Content = sanitizer.Sanitize(item.Link, item.Content)
for i, link := range item.MediaLinks {
item.MediaLinks[i].Description = sanitizer.Sanitize(item.Link, link.Description)
}
c.JSON(http.StatusOK, item)
} else if c.Req.Method == "PUT" {

41
src/storage/item.go
View File

@ -1,6 +1,7 @@
package storage
import (
"database/sql/driver"
"encoding/json"
"fmt"
"log"
@ -44,6 +45,25 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error {
return nil
}
type MediaLink struct {
URL string `json:"url"`
Type string `json:"type"`
Description string `json:"description,omitempty"`
}
type MediaLinks []MediaLink
func (m *MediaLinks) Scan(src any) error {
if data, ok := src.([]byte); ok {
return json.Unmarshal(data, m)
}
return nil
}
func (m MediaLinks) Value() (driver.Value, error) {
return json.Marshal(m)
}
type Item struct {
Id int64 `json:"id"`
GUID string `json:"guid"`
@ -53,8 +73,7 @@ type Item struct {
Content string `json:"content,omitempty"`
Date time.Time `json:"date"`
Status ItemStatus `json:"status"`
ImageURL *string `json:"image"`
AudioURL *string `json:"podcast_url"`
MediaLinks MediaLinks `json:"media_links"`
}
type ItemFilter struct {
@ -110,13 +129,17 @@ func (s *Storage) CreateItems(items []Item) bool {
_, err = tx.Exec(`
insert into items (
guid, feed_id, title, link, date,
content, image, podcast_url,
content, media_links,
date_arrived, status
)
values (?, ?, ?, ?, strftime('%Y-%m-%d %H:%M:%f', ?), ?, ?, ?, ?, ?)
values (
?, ?, ?, ?, strftime('%Y-%m-%d %H:%M:%f', ?),
?, ?,
?, ?
)
on conflict (feed_id, guid) do nothing`,
item.GUID, item.FeedId, item.Title, item.Link, item.Date,
item.Content, item.ImageURL, item.AudioURL,
item.Content, item.MediaLinks,
now, UNREAD,
)
if err != nil {
@ -231,7 +254,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, with
order = "i.id desc"
}
selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url"
selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.media_links"
if withContent {
selectCols += ", i.content"
} else {
@ -254,7 +277,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, with
err = rows.Scan(
&x.Id, &x.GUID, &x.FeedId,
&x.Title, &x.Link, &x.Date,
&x.Status, &x.ImageURL, &x.AudioURL, &x.Content,
&x.Status, &x.MediaLinks, &x.Content,
)
if err != nil {
log.Print(err)
@ -270,12 +293,12 @@ func (s *Storage) GetItem(id int64) *Item {
err := s.db.QueryRow(`
select
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
i.date, i.status, i.image, i.podcast_url
i.date, i.status, i.media_links
from items i
where i.id = ?
`, id).Scan(
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
&i.Date, &i.Status, &i.ImageURL, &i.AudioURL,
&i.Date, &i.Status, &i.MediaLinks,
)
if err != nil {
log.Print(err)

4
src/storage/item_test.go
View File

@ -77,12 +77,12 @@ func getItem(db *Storage, guid string) *Item {
err := db.db.QueryRow(`
select
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
i.date, i.status, i.image, i.podcast_url
i.date, i.status, i.media_links
from items i
where i.guid = ?
`, guid).Scan(
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
&i.Date, &i.Status, &i.ImageURL, &i.AudioURL,
&i.Date, &i.Status, &i.MediaLinks,
)
if err != nil {
log.Fatal(err)

26
src/storage/migration.go
View File

@ -17,6 +17,7 @@ var migrations = []func(*sql.Tx) error{
m07_add_feed_size,
m08_normalize_datetime,
m09_change_item_index,
m10_add_item_medialinks,
}
var maxVersion = int64(len(migrations))
@ -306,3 +307,28 @@ func m09_change_item_index(tx *sql.Tx) error {
_, err := tx.Exec(sql)
return err
}
func m10_add_item_medialinks(tx *sql.Tx) error {
sql := `
alter table items add column media_links blob;
update items set media_links =
iif(
coalesce(image, '') != '' and coalesce(podcast_url, '') != '',
json_array(json_object('type', 'image', 'url', image), json_object('type', 'audio', 'url', podcast_url)),
iif(
coalesce(image, '') != '',
json_array(json_object('type', 'image', 'url', image)),
iif(
coalesce(podcast_url, '') != '',
json_array(json_object('type', 'audio', 'url', podcast_url)),
null
)
)
);
alter table items drop column image;
alter table items drop column podcast_url;
`
_, err := tx.Exec(sql)
return err
}

13
src/worker/crawler.go
View File

@ -143,13 +143,9 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
result := make([]storage.Item, len(items))
for i, item := range items {
item := item
var audioURL *string = nil
if item.AudioURL != "" {
audioURL = &item.AudioURL
}
var imageURL *string = nil
if item.ImageURL != "" {
imageURL = &item.ImageURL
mediaLinks := make(storage.MediaLinks, 0)
for _, link := range item.MediaLinks {
mediaLinks = append(mediaLinks, storage.MediaLink(link))
}
result[i] = storage.Item{
GUID: item.GUID,
@ -159,8 +155,7 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
Content: item.Content,
Date: item.Date,
Status: storage.UNREAD,
ImageURL: imageURL,
AudioURL: audioURL,
MediaLinks: mediaLinks,
}
}
return result