From b8afa82a81743f1cfbe346bbaf3c3ca9499e9f6d Mon Sep 17 00:00:00 2001 From: Nazar Kanaev Date: Wed, 25 Sep 2024 12:32:23 +0100 Subject: [PATCH] support multiple media links --- doc/changelog.txt | 1 + go.mod | 2 +- makefile | 13 +++--- src/assets/index.html | 10 ++++- src/assets/javascripts/app.js | 12 +++++ src/parser/atom.go | 15 ++++--- src/parser/atom_test.go | 28 +++++++----- src/parser/feed.go | 13 +++--- src/parser/media.go | 66 +++++++++++++++++++++++++--- src/parser/models.go | 11 +++-- src/parser/rss.go | 19 ++++---- src/parser/rss_test.go | 82 ++++++++++++++++++++++++++++++----- src/server/routes.go | 3 ++ src/storage/item.go | 57 ++++++++++++++++-------- src/storage/item_test.go | 4 +- src/storage/migration.go | 26 +++++++++++ src/worker/crawler.go | 27 +++++------- 17 files changed, 291 insertions(+), 98 deletions(-) diff --git a/doc/changelog.txt b/doc/changelog.txt index a31f13b..1c4f025 100644 --- a/doc/changelog.txt +++ b/doc/changelog.txt @@ -3,6 +3,7 @@ - (new) Fever API support (thanks to @icefed) - (new) editable feed link (thanks to @adaszko) - (new) switch to feed by clicking the title in the article page (thanks to @tarasglek for suggestion) +- (new) support multiple media links - (fix) duplicate articles caused by the same feed addition (thanks to @adaszko) - (fix) relative article links (thanks to @adazsko for the report) - (fix) atom article links stored in id element (thanks to @adazsko for the report) diff --git a/go.mod b/go.mod index a257704..2d87be1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/nkanaev/yarr -go 1.17 +go 1.18 require ( github.com/mattn/go-sqlite3 v1.14.7 diff --git a/makefile b/makefile index e0cd718..1d1a974 100644 --- a/makefile +++ b/makefile @@ -1,6 +1,7 @@ VERSION=2.4 GITHASH=$(shell git rev-parse --short=8 HEAD) +GO_TAGS = sqlite_foreign_keys sqlite_json GO_LDFLAGS = -s -w -X 'main.Version=$(VERSION)' -X 'main.GitHash=$(GITHASH)' export GOARCH ?= amd64 @@ -8,26 +9,26 @@ export CGO_ENABLED = 1 build_default: mkdir -p _output - go build -tags "sqlite_foreign_keys" -ldflags="$(GO_LDFLAGS)" -o _output/yarr ./cmd/yarr + go build -tags "$(GO_TAGS)" -ldflags="$(GO_LDFLAGS)" -o _output/yarr ./cmd/yarr build_macos: mkdir -p _output/macos - GOOS=darwin go build -tags "sqlite_foreign_keys macos" -ldflags="$(GO_LDFLAGS)" -o _output/macos/yarr ./cmd/yarr + GOOS=darwin go build -tags "$(GO_TAGS) macos" -ldflags="$(GO_LDFLAGS)" -o _output/macos/yarr ./cmd/yarr cp src/platform/icon.png _output/macos/icon.png go run ./cmd/package_macos -outdir _output/macos -version "$(VERSION)" build_linux: mkdir -p _output/linux - GOOS=linux go build -tags "sqlite_foreign_keys linux" -ldflags="$(GO_LDFLAGS)" -o _output/linux/yarr ./cmd/yarr + GOOS=linux go build -tags "$(GO_TAGS) linux" -ldflags="$(GO_LDFLAGS)" -o _output/linux/yarr ./cmd/yarr build_windows: mkdir -p _output/windows go run ./cmd/generate_versioninfo -version "$(VERSION)" -outfile src/platform/versioninfo.rc windres -i src/platform/versioninfo.rc -O coff -o src/platform/versioninfo.syso - GOOS=windows go build -tags "sqlite_foreign_keys windows" -ldflags="$(GO_LDFLAGS) -H windowsgui" -o _output/windows/yarr.exe ./cmd/yarr + GOOS=windows go build -tags "$(GO_TAGS) windows" -ldflags="$(GO_LDFLAGS) -H windowsgui" -o _output/windows/yarr.exe ./cmd/yarr serve: - go run -tags "sqlite_foreign_keys" ./cmd/yarr -db local.db + go run -tags "$(GO_TAGS)" ./cmd/yarr -db local.db test: - go test -tags "sqlite_foreign_keys" ./... + go test -tags "$(GO_TAGS)" ./... diff --git a/src/assets/index.html b/src/assets/index.html index 59f3776..c3ab2ed 100644 --- a/src/assets/index.html +++ b/src/assets/index.html @@ -362,8 +362,14 @@
- - +
+
+ +
+
+
+ +
diff --git a/src/assets/javascripts/app.js b/src/assets/javascripts/app.js index 6e187ac..38056c5 100644 --- a/src/assets/javascripts/app.js +++ b/src/assets/javascripts/app.js @@ -298,6 +298,18 @@ var vm = new Vue({ return this.itemSelectedDetails.content || '' }, + contentImages: function() { + if (!this.itemSelectedDetails) return [] + return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'image') + }, + contentAudios: function() { + if (!this.itemSelectedDetails) return [] + return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'audio') + }, + contentVideos: function() { + if (!this.itemSelectedDetails) return [] + return (this.itemSelectedDetails.media_links || []).filter(l => l.type === 'video') + } }, watch: { 'theme': { diff --git a/src/parser/atom.go b/src/parser/atom.go index 5bc0c57..607aa04 100644 --- a/src/parser/atom.go +++ b/src/parser/atom.go @@ -89,15 +89,16 @@ func ParseAtom(r io.Reader) (*Feed, error) { guidFromID = srcitem.ID + "::" + srcitem.Updated } + mediaLinks := srcitem.mediaLinks() + link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""), linkFromID) dstfeed.Items = append(dstfeed.Items, Item{ - GUID: firstNonEmpty(guidFromID, srcitem.ID, link), - Date: dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)), - URL: link, - Title: srcitem.Title.Text(), - Content: firstNonEmpty(srcitem.Content.String(), srcitem.Summary.String(), srcitem.firstMediaDescription()), - ImageURL: srcitem.firstMediaThumbnail(), - AudioURL: "", + GUID: firstNonEmpty(guidFromID, srcitem.ID, link), + Date: dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)), + URL: link, + Title: srcitem.Title.Text(), + Content: firstNonEmpty(srcitem.Content.String(), srcitem.Summary.String(), srcitem.firstMediaDescription()), + MediaLinks: mediaLinks, }) } return dstfeed, nil diff --git a/src/parser/atom_test.go b/src/parser/atom_test.go index 88b40a3..3725b14 100644 --- a/src/parser/atom_test.go +++ b/src/parser/atom_test.go @@ -40,13 +40,11 @@ func TestAtom(t *testing.T) { SiteURL: "http://example.org/", Items: []Item{ { - GUID: "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a", - Date: time.Unix(1071340202, 0).UTC(), - URL: "http://example.org/2003/12/13/atom03.html", - Title: "Atom-Powered Robots Run Amok", - Content: `

This is the entry content.

`, - ImageURL: "", - AudioURL: "", + GUID: "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a", + Date: time.Unix(1071340202, 0).UTC(), + URL: "http://example.org/2003/12/13/atom03.html", + Title: "Atom-Powered Robots Run Amok", + Content: `

This is the entry content.

`, }, }, } @@ -141,9 +139,15 @@ func TestAtomImageLink(t *testing.T) { `)) - have := feed.Items[0].ImageURL - want := `https://example.com/image.png?width=100&height=100` - if want != have { + if len(feed.Items[0].MediaLinks) != 1 { + t.Fatalf("Expected 1 media link, got: %#v", feed.Items[0].MediaLinks) + } + have := feed.Items[0].MediaLinks[0] + want := MediaLink{ + URL: `https://example.com/image.png?width=100&height=100`, + Type: "image", + } + if !reflect.DeepEqual(want, have) { t.Fatalf("item.image_url doesn't match\nwant: %#v\nhave: %#v\n", want, have) } } @@ -165,8 +169,8 @@ func TestAtomImageLinkDuplicated(t *testing.T) { if want != have { t.Fatalf("want: %#v\nhave: %#v\n", want, have) } - if feed.Items[0].ImageURL != "" { - t.Fatal("item.image_url must be unset if present in the content") + if len(feed.Items[0].MediaLinks) != 0 { + t.Fatal("item media link must be excluded if present in the content") } } diff --git a/src/parser/feed.go b/src/parser/feed.go index e47a873..a735127 100644 --- a/src/parser/feed.go +++ b/src/parser/feed.go @@ -134,11 +134,14 @@ func (feed *Feed) cleanup() { feed.Items[i].Title = strings.TrimSpace(htmlutil.ExtractText(item.Title)) feed.Items[i].Content = strings.TrimSpace(item.Content) - if item.ImageURL != "" && strings.Contains(item.Content, item.ImageURL) { - feed.Items[i].ImageURL = "" - } - if item.AudioURL != "" && strings.Contains(item.Content, item.AudioURL) { - feed.Items[i].AudioURL = "" + if len(feed.Items[i].MediaLinks) > 0 { + mediaLinks := make([]MediaLink, 0) + for _, link := range item.MediaLinks { + if !strings.Contains(item.Content, link.URL) { + mediaLinks = append(mediaLinks, link) + } + } + feed.Items[i].MediaLinks = mediaLinks } } } diff --git a/src/parser/media.go b/src/parser/media.go index b926d1e..df7db98 100644 --- a/src/parser/media.go +++ b/src/parser/media.go @@ -1,5 +1,9 @@ package parser +import ( + "strings" +) + type media struct { MediaGroups []mediaGroup `xml:"http://search.yahoo.com/mrss/ group"` MediaContents []mediaContent `xml:"http://search.yahoo.com/mrss/ content"` @@ -8,12 +12,17 @@ type media struct { } type mediaGroup struct { + MediaContent []mediaContent `xml:"http://search.yahoo.com/mrss/ content"` MediaThumbnails []mediaThumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"` MediaDescriptions []mediaDescription `xml:"http://search.yahoo.com/mrss/ description"` } type mediaContent struct { - MediaThumbnails []mediaThumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"` + MediaThumbnails []mediaThumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"` + MediaType string `xml:"type,attr"` + MediaMedium string `xml:"medium,attr"` + MediaURL string `xml:"url,attr"` + MediaDescription mediaDescription `xml:"http://search.yahoo.com/mrss/ description"` } type mediaThumbnail struct { @@ -21,8 +30,8 @@ type mediaThumbnail struct { } type mediaDescription struct { - Type string `xml:"type,attr"` - Description string `xml:",chardata"` + Type string `xml:"type,attr"` + Text string `xml:",chardata"` } func (m *media) firstMediaThumbnail() string { @@ -44,12 +53,59 @@ func (m *media) firstMediaThumbnail() string { func (m *media) firstMediaDescription() string { for _, d := range m.MediaDescriptions { - return plain2html(d.Description) + return plain2html(d.Text) } for _, g := range m.MediaGroups { for _, d := range g.MediaDescriptions { - return plain2html(d.Description) + return plain2html(d.Text) } } return "" } + +func (m *media) mediaLinks() []MediaLink { + links := make([]MediaLink, 0) + for _, thumbnail := range m.MediaThumbnails { + links = append(links, MediaLink{URL: thumbnail.URL, Type: "image"}) + } + for _, group := range m.MediaGroups { + for _, thumbnail := range group.MediaThumbnails { + links = append(links, MediaLink{ + URL: thumbnail.URL, + Type: "image", + }) + } + } + for _, content := range m.MediaContents { + if content.MediaURL != "" { + url := content.MediaURL + description := content.MediaDescription.Text + if strings.HasPrefix(content.MediaType, "image/") { + links = append(links, MediaLink{URL: url, Type: "image", Description: description}) + } else if strings.HasPrefix(content.MediaType, "audio/") { + links = append(links, MediaLink{URL: url, Type: "audio", Description: description}) + } else if strings.HasPrefix(content.MediaType, "video/") { + links = append(links, MediaLink{URL: url, Type: "video", Description: description}) + } else if content.MediaMedium == "image" || content.MediaMedium == "audio" || content.MediaMedium == "video" { + links = append(links, MediaLink{URL: url, Type: content.MediaMedium, Description: description}) + } else { + if len(content.MediaThumbnails) > 0 { + links = append(links, MediaLink{ + URL: content.MediaThumbnails[0].URL, + Type: "image", + }) + } + } + } + for _, thumbnail := range content.MediaThumbnails { + links = append(links, MediaLink{ + URL: thumbnail.URL, + Type: "image", + }) + } + } + if len(links) == 0 { + return nil + } + return links +} diff --git a/src/parser/models.go b/src/parser/models.go index 7587b77..a756739 100644 --- a/src/parser/models.go +++ b/src/parser/models.go @@ -14,7 +14,12 @@ type Item struct { URL string Title string - Content string - ImageURL string - AudioURL string + Content string + MediaLinks []MediaLink +} + +type MediaLink struct { + URL string + Type string + Description string } diff --git a/src/parser/rss.go b/src/parser/rss.go index 22090db..0e43e13 100644 --- a/src/parser/rss.go +++ b/src/parser/rss.go @@ -74,14 +74,14 @@ func ParseRSS(r io.Reader) (*Feed, error) { SiteURL: srcfeed.Link, } for _, srcitem := range srcfeed.Items { - podcastURL := "" + mediaLinks := srcitem.mediaLinks() for _, e := range srcitem.Enclosures { if strings.HasPrefix(e.Type, "audio/") { - podcastURL = e.URL - + podcastURL := e.URL if srcitem.OrigEnclosureLink != "" && strings.Contains(podcastURL, path.Base(srcitem.OrigEnclosureLink)) { podcastURL = srcitem.OrigEnclosureLink } + mediaLinks = append(mediaLinks, MediaLink{URL: podcastURL, Type: "audio"}) break } } @@ -92,13 +92,12 @@ func ParseRSS(r io.Reader) (*Feed, error) { } dstfeed.Items = append(dstfeed.Items, Item{ - GUID: firstNonEmpty(srcitem.GUID.GUID, srcitem.Link), - Date: dateParse(firstNonEmpty(srcitem.DublinCoreDate, srcitem.PubDate)), - URL: firstNonEmpty(srcitem.OrigLink, srcitem.Link, permalink), - Title: srcitem.Title, - Content: firstNonEmpty(srcitem.ContentEncoded, srcitem.Description), - AudioURL: podcastURL, - ImageURL: srcitem.firstMediaThumbnail(), + GUID: firstNonEmpty(srcitem.GUID.GUID, srcitem.Link), + Date: dateParse(firstNonEmpty(srcitem.DublinCoreDate, srcitem.PubDate)), + URL: firstNonEmpty(srcitem.OrigLink, srcitem.Link, permalink), + Title: srcitem.Title, + Content: firstNonEmpty(srcitem.ContentEncoded, srcitem.Description, srcitem.firstMediaDescription()), + MediaLinks: mediaLinks, }) } return dstfeed, nil diff --git a/src/parser/rss_test.go b/src/parser/rss_test.go index e1d5e67..68d1ae3 100644 --- a/src/parser/rss_test.go +++ b/src/parser/rss_test.go @@ -75,9 +75,15 @@ func TestRSSMediaContentThumbnail(t *testing.T) { `)) - have := feed.Items[0].ImageURL - want := "https://i.vimeocdn.com/video/1092705247_960.jpg" - if have != want { + if len(feed.Items[0].MediaLinks) != 1 { + t.Fatalf("Expected 1 media link, got %#v", feed.Items[0].MediaLinks) + } + have := feed.Items[0].MediaLinks[0] + want := MediaLink{ + URL: "https://i.vimeocdn.com/video/1092705247_960.jpg", + Type: "image", + } + if !reflect.DeepEqual(want, have) { t.Logf("want: %#v", want) t.Logf("have: %#v", have) t.FailNow() @@ -127,9 +133,15 @@ func TestRSSPodcast(t *testing.T) { `)) - have := feed.Items[0].AudioURL - want := "http://example.com/audio.ext" - if want != have { + if len(feed.Items[0].MediaLinks) != 1 { + t.Fatal("Invalid media links") + } + have := feed.Items[0].MediaLinks[0] + want := MediaLink{ + URL: "http://example.com/audio.ext", + Type: "audio", + } + if !reflect.DeepEqual(want, have) { t.Logf("want: %#v", want) t.Logf("have: %#v", have) t.FailNow() @@ -147,9 +159,15 @@ func TestRSSOpusPodcast(t *testing.T) { `)) - have := feed.Items[0].AudioURL - want := "http://example.com/audio.ext" - if want != have { + if len(feed.Items[0].MediaLinks) != 1 { + t.Fatal("Invalid media links") + } + have := feed.Items[0].MediaLinks[0] + want := MediaLink{ + URL: "http://example.com/audio.ext", + Type: "audio", + } + if !reflect.DeepEqual(want, have) { t.Logf("want: %#v", want) t.Logf("have: %#v", have) t.FailNow() @@ -176,8 +194,9 @@ func TestRSSPodcastDuplicated(t *testing.T) { if want != have { t.Fatalf("content doesn't match\nwant: %#v\nhave: %#v\n", want, have) } - if feed.Items[0].AudioURL != "" { - t.Fatal("item.audio_url must be unset if present in the content") + + if len(feed.Items[0].MediaLinks) != 0 { + t.Fatal("item media must be excluded if present in the content") } } @@ -223,8 +242,47 @@ func TestRSSIsPermalink(t *testing.T) { }, } for i := 0; i < len(want); i++ { - if want[i] != have[i] { + if !reflect.DeepEqual(want, have) { t.Errorf("Failed to handle isPermalink\nwant: %#v\nhave: %#v\n", want[i], have[i]) } } } + +func TestRSSMultipleMedia(t *testing.T) { + feed, _ := Parse(strings.NewReader(` + + + + + http://example.com/posts/1 + + description 1 + + + description 2 + + + video description + + + + + `)) + have := feed.Items + want := []Item{ + { + GUID: "http://example.com/posts/1", + URL: "http://example.com/posts/1", + MediaLinks: []MediaLink{ + {URL:"https://example.com/path/to/image1.png", Type:"image", Description:"description 1"}, + {URL:"https://example.com/path/to/image2.png", Type:"image", Description:"description 2"}, + {URL:"https://example.com/path/to/video1.mp4", Type:"video", Description:"video description"}, + }, + }, + } + if !reflect.DeepEqual(want, have) { + t.Logf("want: %#v", want) + t.Logf("have: %#v", have) + t.Fatal("invalid rss") + } +} diff --git a/src/server/routes.go b/src/server/routes.go index 2398d4d..ad4e4bd 100644 --- a/src/server/routes.go +++ b/src/server/routes.go @@ -329,6 +329,9 @@ func (s *Server) handleItem(c *router.Context) { } item.Content = sanitizer.Sanitize(item.Link, item.Content) + for i, link := range item.MediaLinks { + item.MediaLinks[i].Description = sanitizer.Sanitize(item.Link, link.Description) + } c.JSON(http.StatusOK, item) } else if c.Req.Method == "PUT" { diff --git a/src/storage/item.go b/src/storage/item.go index 68c6821..9af1197 100644 --- a/src/storage/item.go +++ b/src/storage/item.go @@ -1,6 +1,7 @@ package storage import ( + "database/sql/driver" "encoding/json" "fmt" "log" @@ -44,17 +45,35 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error { return nil } +type MediaLink struct { + URL string `json:"url"` + Type string `json:"type"` + Description string `json:"description,omitempty"` +} + +type MediaLinks []MediaLink + +func (m *MediaLinks) Scan(src any) error { + if data, ok := src.([]byte); ok { + return json.Unmarshal(data, m) + } + return nil +} + +func (m MediaLinks) Value() (driver.Value, error) { + return json.Marshal(m) +} + type Item struct { - Id int64 `json:"id"` - GUID string `json:"guid"` - FeedId int64 `json:"feed_id"` - Title string `json:"title"` - Link string `json:"link"` - Content string `json:"content,omitempty"` - Date time.Time `json:"date"` - Status ItemStatus `json:"status"` - ImageURL *string `json:"image"` - AudioURL *string `json:"podcast_url"` + Id int64 `json:"id"` + GUID string `json:"guid"` + FeedId int64 `json:"feed_id"` + Title string `json:"title"` + Link string `json:"link"` + Content string `json:"content,omitempty"` + Date time.Time `json:"date"` + Status ItemStatus `json:"status"` + MediaLinks MediaLinks `json:"media_links"` } type ItemFilter struct { @@ -110,13 +129,17 @@ func (s *Storage) CreateItems(items []Item) bool { _, err = tx.Exec(` insert into items ( guid, feed_id, title, link, date, - content, image, podcast_url, + content, media_links, date_arrived, status ) - values (?, ?, ?, ?, strftime('%Y-%m-%d %H:%M:%f', ?), ?, ?, ?, ?, ?) + values ( + ?, ?, ?, ?, strftime('%Y-%m-%d %H:%M:%f', ?), + ?, ?, + ?, ? + ) on conflict (feed_id, guid) do nothing`, item.GUID, item.FeedId, item.Title, item.Link, item.Date, - item.Content, item.ImageURL, item.AudioURL, + item.Content, item.MediaLinks, now, UNREAD, ) if err != nil { @@ -231,7 +254,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, with order = "i.id desc" } - selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url" + selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.media_links" if withContent { selectCols += ", i.content" } else { @@ -254,7 +277,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, with err = rows.Scan( &x.Id, &x.GUID, &x.FeedId, &x.Title, &x.Link, &x.Date, - &x.Status, &x.ImageURL, &x.AudioURL, &x.Content, + &x.Status, &x.MediaLinks, &x.Content, ) if err != nil { log.Print(err) @@ -270,12 +293,12 @@ func (s *Storage) GetItem(id int64) *Item { err := s.db.QueryRow(` select i.id, i.guid, i.feed_id, i.title, i.link, i.content, - i.date, i.status, i.image, i.podcast_url + i.date, i.status, i.media_links from items i where i.id = ? `, id).Scan( &i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content, - &i.Date, &i.Status, &i.ImageURL, &i.AudioURL, + &i.Date, &i.Status, &i.MediaLinks, ) if err != nil { log.Print(err) diff --git a/src/storage/item_test.go b/src/storage/item_test.go index e227eba..88fc63d 100644 --- a/src/storage/item_test.go +++ b/src/storage/item_test.go @@ -77,12 +77,12 @@ func getItem(db *Storage, guid string) *Item { err := db.db.QueryRow(` select i.id, i.guid, i.feed_id, i.title, i.link, i.content, - i.date, i.status, i.image, i.podcast_url + i.date, i.status, i.media_links from items i where i.guid = ? `, guid).Scan( &i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content, - &i.Date, &i.Status, &i.ImageURL, &i.AudioURL, + &i.Date, &i.Status, &i.MediaLinks, ) if err != nil { log.Fatal(err) diff --git a/src/storage/migration.go b/src/storage/migration.go index 1f33967..2dac555 100644 --- a/src/storage/migration.go +++ b/src/storage/migration.go @@ -17,6 +17,7 @@ var migrations = []func(*sql.Tx) error{ m07_add_feed_size, m08_normalize_datetime, m09_change_item_index, + m10_add_item_medialinks, } var maxVersion = int64(len(migrations)) @@ -306,3 +307,28 @@ func m09_change_item_index(tx *sql.Tx) error { _, err := tx.Exec(sql) return err } + +func m10_add_item_medialinks(tx *sql.Tx) error { + sql := ` + alter table items add column media_links blob; + update items set media_links = + iif( + coalesce(image, '') != '' and coalesce(podcast_url, '') != '', + json_array(json_object('type', 'image', 'url', image), json_object('type', 'audio', 'url', podcast_url)), + iif( + coalesce(image, '') != '', + json_array(json_object('type', 'image', 'url', image)), + iif( + coalesce(podcast_url, '') != '', + json_array(json_object('type', 'audio', 'url', podcast_url)), + null + ) + ) + ); + + alter table items drop column image; + alter table items drop column podcast_url; + ` + _, err := tx.Exec(sql) + return err +} diff --git a/src/worker/crawler.go b/src/worker/crawler.go index 3720a38..ebefaa1 100644 --- a/src/worker/crawler.go +++ b/src/worker/crawler.go @@ -143,24 +143,19 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item { result := make([]storage.Item, len(items)) for i, item := range items { item := item - var audioURL *string = nil - if item.AudioURL != "" { - audioURL = &item.AudioURL - } - var imageURL *string = nil - if item.ImageURL != "" { - imageURL = &item.ImageURL + mediaLinks := make(storage.MediaLinks, 0) + for _, link := range item.MediaLinks { + mediaLinks = append(mediaLinks, storage.MediaLink(link)) } result[i] = storage.Item{ - GUID: item.GUID, - FeedId: feed.Id, - Title: item.Title, - Link: item.URL, - Content: item.Content, - Date: item.Date, - Status: storage.UNREAD, - ImageURL: imageURL, - AudioURL: audioURL, + GUID: item.GUID, + FeedId: feed.Id, + Title: item.Title, + Link: item.URL, + Content: item.Content, + Date: item.Date, + Status: storage.UNREAD, + MediaLinks: mediaLinks, } } return result