diff --git a/doc/changelog.md b/doc/changelog.md index 6c379f6..67bc52a 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -1,6 +1,7 @@ # upcoming - (fix) support for HTTP/2 (thanks to @Dean-Corso for the report) +- (fix) displaying correct title for custom Youtube feeds - (etc) do not auto-refresh feeds on startup # v2.7 (2026-06-24) diff --git a/src/content/scraper/finder.go b/src/content/scraper/finder.go index 9f782ec..fb32800 100644 --- a/src/content/scraper/finder.go +++ b/src/content/scraper/finder.go @@ -1,6 +1,7 @@ package scraper import ( + "maps" "net/url" "slices" "strings" @@ -9,12 +10,18 @@ import ( "golang.org/x/net/html" ) -func FindFeeds(body string, base string) map[string]string { - candidates := make(map[string]string) +type FeedLink struct { + URL string `json:"url"` + Title string `json:"title"` + TitleOverride string `json:"title_override,omitempty"` +} + +func FindFeeds(body string, base string) []FeedLink { + candidates := make(map[string]FeedLink) doc, err := html.Parse(strings.NewReader(body)) if err != nil { - return candidates + return nil } // find direct links @@ -34,17 +41,48 @@ func FindFeeds(body string, base string) map[string]string { name := htmlutil.Attr(node, "title") link := htmlutil.AbsoluteUrl(href, base) if link != "" { - candidates[link] = name + candidates[link] = FeedLink{URL: link, Title: name} l, err := url.Parse(link) if err == nil && l.Host == "www.youtube.com" && l.Path == "/feeds/videos.xml" { // https://wiki.archiveteam.org/index.php/YouTube/Technical_details#Playlists channelID, found := strings.CutPrefix(l.Query().Get("channel_id"), "UC") if found { - const url string = "https://www.youtube.com/feeds/videos.xml?playlist_id=" - candidates[url+"UULF"+channelID] = name + " - Videos" - candidates[url+"UULV"+channelID] = name + " - Live Streams" - candidates[url+"UUSH"+channelID] = name + " - Short videos" + const baseURL string = "https://www.youtube.com/feeds/videos.xml?playlist_id=" + + ogTitle := "" + isOG := func(n *html.Node) bool { + return n.Type == html.ElementNode && n.Data == "meta" && + htmlutil.Attr(n, "property") == "og:title" + } + for _, n := range htmlutil.FindNodes(doc, isOG) { + ogTitle = htmlutil.Attr(n, "content") + break + } + override := name + if ogTitle != "" { + override = ogTitle + } + + candidates[link] = FeedLink{ + URL: link, + Title: name + " - All", + } + candidates[baseURL+"UULF"+channelID] = FeedLink{ + URL: baseURL + "UULF" + channelID, + Title: name + " - Videos", + TitleOverride: override + " - Videos", + } + candidates[baseURL+"UULV"+channelID] = FeedLink{ + URL: baseURL + "UULV" + channelID, + Title: name + " - Live Streams", + TitleOverride: override + " - Live Streams", + } + candidates[baseURL+"UUSH"+channelID] = FeedLink{ + URL: baseURL + "UUSH" + channelID, + Title: name + " - Short videos", + TitleOverride: override + " - Short videos", + } } } } @@ -77,12 +115,19 @@ func FindFeeds(body string, base string) map[string]string { href := htmlutil.Attr(node, "href") link := htmlutil.AbsoluteUrl(href, base) if link != "" { - candidates[link] = "" + candidates[link] = FeedLink{URL: link, Title: ""} } } } - return candidates + result := slices.Collect(maps.Values(candidates)) + slices.SortFunc(result, func(a, b FeedLink) int { + if a.Title != b.Title { + return strings.Compare(a.Title, b.Title) + } + return strings.Compare(a.URL, b.URL) + }) + return result } func FindIcons(body string, base string) []string { diff --git a/src/content/scraper/finder_test.go b/src/content/scraper/finder_test.go index 3b48333..4c99016 100644 --- a/src/content/scraper/finder_test.go +++ b/src/content/scraper/finder_test.go @@ -33,10 +33,10 @@ func TestFindFeedsLinks(t *testing.T) { ` have := FindFeeds(x, base) - want := map[string]string{ - base + "/feed.xml": "rss with title", - base + "/atom.xml": "", - base + "/feed.json": "", + want := []FeedLink{ + {URL: base + "/atom.xml", Title: ""}, + {URL: base + "/feed.json", Title: ""}, + {URL: base + "/feed.xml", Title: "rss with title"}, } if !reflect.DeepEqual(have, want) { t.Logf("want: %#v", want) @@ -61,9 +61,9 @@ func TestFindFeedsGuess(t *testing.T) { ` have := FindFeeds(body, base) - want := map[string]string{ - base + "/feed.xml": "", - base + "/news": "", + want := []FeedLink{ + {URL: base + "/feed.xml", Title: ""}, + {URL: base + "/news", Title: ""}, } if !reflect.DeepEqual(want, have) { t.Logf("want: %#v", want) @@ -72,6 +72,110 @@ func TestFindFeedsGuess(t *testing.T) { } } +func TestFindFeedsYouTubeOGTitle(t *testing.T) { + body := ` + + + + + + + + + + ` + have := FindFeeds(body, base) + + youtubeURL := "https://www.youtube.com/feeds/videos.xml?playlist_id=" + want := []FeedLink{ + {URL: "https://www.youtube.com/feeds/videos.xml?channel_id=UCabc123", Title: "YouTube Channel - All"}, + {URL: youtubeURL + "UULVabc123", Title: "YouTube Channel - Live Streams", TitleOverride: "My Channel - Live Streams"}, + {URL: youtubeURL + "UUSHabc123", Title: "YouTube Channel - Short videos", TitleOverride: "My Channel - Short videos"}, + {URL: youtubeURL + "UULFabc123", Title: "YouTube Channel - Videos", TitleOverride: "My Channel - Videos"}, + } + if !reflect.DeepEqual(have, want) { + t.Logf("want: %#v", want) + t.Logf("have: %#v", have) + t.Fatal("invalid result") + } +} + +func TestFindFeedsYouTubeNoOGTitle(t *testing.T) { + body := ` + + + + + + + + + ` + have := FindFeeds(body, base) + + youtubeURL := "https://www.youtube.com/feeds/videos.xml?playlist_id=" + want := []FeedLink{ + {URL: "https://www.youtube.com/feeds/videos.xml?channel_id=UCxyz789", Title: "Channel Name - All"}, + {URL: youtubeURL + "UULVxyz789", Title: "Channel Name - Live Streams", TitleOverride: "Channel Name - Live Streams"}, + {URL: youtubeURL + "UUSHxyz789", Title: "Channel Name - Short videos", TitleOverride: "Channel Name - Short videos"}, + {URL: youtubeURL + "UULFxyz789", Title: "Channel Name - Videos", TitleOverride: "Channel Name - Videos"}, + } + if !reflect.DeepEqual(have, want) { + t.Logf("want: %#v", want) + t.Logf("have: %#v", have) + t.Fatal("invalid result") + } +} + +func TestFindFeedsYouTubeNoChannelID(t *testing.T) { + body := ` + + + + + + + + + ` + have := FindFeeds(body, base) + + want := []FeedLink{ + {URL: "https://www.youtube.com/feeds/videos.xml?channel_id=invalid", Title: "Youtube"}, + } + if !reflect.DeepEqual(have, want) { + t.Logf("want: %#v", want) + t.Logf("have: %#v", have) + t.Fatal("invalid result") + } +} + +func TestFindFeedsNonYouTubeNoTitleOverride(t *testing.T) { + body := ` + + + + + + + + + ` + have := FindFeeds(body, base) + + want := []FeedLink{ + {URL: base + "/blog.xml", Title: "Blog"}, + } + if !reflect.DeepEqual(have, want) { + t.Logf("want: %#v", want) + t.Logf("have: %#v", have) + t.Fatal("invalid result") + } + if have[0].TitleOverride != "" { + t.Fatal("expected empty TitleOverride for non-YouTube feed") + } +} + func TestFindIcons(t *testing.T) { body := ` diff --git a/src/frontend/js/app.ts b/src/frontend/js/app.ts index f4b1661..0730d89 100644 --- a/src/frontend/js/app.ts +++ b/src/frontend/js/app.ts @@ -459,14 +459,16 @@ export default { }) } }, - createFeed: function(event) { - var form = event.target + createFeed: function($event) { + var form = $event.target var data = { url: form.querySelector('input[name=url]').value, folder_id: parseInt(form.querySelector('select[name=folder_id]').value) || null, } if (this.feedNewChoiceSelected) { + var choice = this.feedNewChoice.find(c => c.url === this.feedNewChoiceSelected) data.url = this.feedNewChoiceSelected + if (choice && choice.title_override) data.title_override = choice.title_override } this.loading.newfeed = true api.feeds.create(data).then(function(result) { diff --git a/src/frontend/js/templates/index.html b/src/frontend/js/templates/index.html index 594d014..8bafa21 100644 --- a/src/frontend/js/templates/index.html +++ b/src/frontend/js/templates/index.html @@ -375,7 +375,7 @@

{{ $t('new_feed') }}

-
+