james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2026-07-15 11:06:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

generate proper titles for Youtube feeds

Browse Source
This commit is contained in:
nkanaev
2026-07-10 23:14:41 +01:00
parent 4647afab6f
commit 56547c5e65
9 changed files with 259 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
doc/changelog.md
View File

@@ -1,6 +1,7 @@
# upcoming
- (fix) support for HTTP/2 (thanks to @Dean-Corso for the report)
- (fix) displaying correct title for custom Youtube feeds
- (etc) do not auto-refresh feeds on startup
# v2.7 (2026-06-24)

65
src/content/scraper/finder.go
View File

@@ -1,6 +1,7 @@
package scraper
import (
"maps"
"net/url"
"slices"
"strings"
@@ -9,12 +10,18 @@ import (
"golang.org/x/net/html"
)
func FindFeeds(body string, base string) map[string]string {
candidates := make(map[string]string)
type FeedLink struct {
URL string `json:"url"`
Title string `json:"title"`
TitleOverride string `json:"title_override,omitempty"`
}
func FindFeeds(body string, base string) []FeedLink {
candidates := make(map[string]FeedLink)
doc, err := html.Parse(strings.NewReader(body))
if err != nil {
return candidates
return nil
}
// find direct links
@@ -34,17 +41,48 @@ func FindFeeds(body string, base string) map[string]string {
name := htmlutil.Attr(node, "title")
link := htmlutil.AbsoluteUrl(href, base)
if link != "" {
candidates[link] = name
candidates[link] = FeedLink{URL: link, Title: name}
l, err := url.Parse(link)
if err == nil && l.Host == "www.youtube.com" && l.Path == "/feeds/videos.xml" {
// https://wiki.archiveteam.org/index.php/YouTube/Technical_details#Playlists
channelID, found := strings.CutPrefix(l.Query().Get("channel_id"), "UC")
if found {
const url string = "https://www.youtube.com/feeds/videos.xml?playlist_id="
candidates[url+"UULF"+channelID] = name + " - Videos"
candidates[url+"UULV"+channelID] = name + " - Live Streams"
candidates[url+"UUSH"+channelID] = name + " - Short videos"
const baseURL string = "https://www.youtube.com/feeds/videos.xml?playlist_id="
ogTitle := ""
isOG := func(n *html.Node) bool {
return n.Type == html.ElementNode && n.Data == "meta" &&
htmlutil.Attr(n, "property") == "og:title"
}
for _, n := range htmlutil.FindNodes(doc, isOG) {
ogTitle = htmlutil.Attr(n, "content")
break
}
override := name
if ogTitle != "" {
override = ogTitle
}
candidates[link] = FeedLink{
URL: link,
Title: name + " - All",
}
candidates[baseURL+"UULF"+channelID] = FeedLink{
URL: baseURL + "UULF" + channelID,
Title: name + " - Videos",
TitleOverride: override + " - Videos",
}
candidates[baseURL+"UULV"+channelID] = FeedLink{
URL: baseURL + "UULV" + channelID,
Title: name + " - Live Streams",
TitleOverride: override + " - Live Streams",
}
candidates[baseURL+"UUSH"+channelID] = FeedLink{
URL: baseURL + "UUSH" + channelID,
Title: name + " - Short videos",
TitleOverride: override + " - Short videos",
}
}
}
}
@@ -77,12 +115,19 @@ func FindFeeds(body string, base string) map[string]string {
href := htmlutil.Attr(node, "href")
link := htmlutil.AbsoluteUrl(href, base)
if link != "" {
candidates[link] = ""
candidates[link] = FeedLink{URL: link, Title: ""}
}
}
}
return candidates
result := slices.Collect(maps.Values(candidates))
slices.SortFunc(result, func(a, b FeedLink) int {
if a.Title != b.Title {
return strings.Compare(a.Title, b.Title)
}
return strings.Compare(a.URL, b.URL)
})
return result
}
func FindIcons(body string, base string) []string {

118
src/content/scraper/finder_test.go
View File

@@ -33,10 +33,10 @@ func TestFindFeedsLinks(t *testing.T) {
`
have := FindFeeds(x, base)
want := map[string]string{
base + "/feed.xml": "rss with title",
base + "/atom.xml": "",
base + "/feed.json": "",
want := []FeedLink{
{URL: base + "/atom.xml", Title: ""},
{URL: base + "/feed.json", Title: ""},
{URL: base + "/feed.xml", Title: "rss with title"},
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -61,9 +61,9 @@ func TestFindFeedsGuess(t *testing.T) {
</html>
`
have := FindFeeds(body, base)
want := map[string]string{
base + "/feed.xml": "",
base + "/news": "",
want := []FeedLink{
{URL: base + "/feed.xml", Title: ""},
{URL: base + "/news", Title: ""},
}
if !reflect.DeepEqual(want, have) {
t.Logf("want: %#v", want)
@@ -72,6 +72,110 @@ func TestFindFeedsGuess(t *testing.T) {
}
}
func TestFindFeedsYouTubeOGTitle(t *testing.T) {
body := `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta property="og:title" content="My Channel">
<link rel="alternate" href="https://www.youtube.com/feeds/videos.xml?channel_id=UCabc123" type="application/rss+xml" title="YouTube Channel">
</head>
<body></body>
</html>
`
have := FindFeeds(body, base)
youtubeURL := "https://www.youtube.com/feeds/videos.xml?playlist_id="
want := []FeedLink{
{URL: "https://www.youtube.com/feeds/videos.xml?channel_id=UCabc123", Title: "YouTube Channel - All"},
{URL: youtubeURL + "UULVabc123", Title: "YouTube Channel - Live Streams", TitleOverride: "My Channel - Live Streams"},
{URL: youtubeURL + "UUSHabc123", Title: "YouTube Channel - Short videos", TitleOverride: "My Channel - Short videos"},
{URL: youtubeURL + "UULFabc123", Title: "YouTube Channel - Videos", TitleOverride: "My Channel - Videos"},
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fatal("invalid result")
}
}
func TestFindFeedsYouTubeNoOGTitle(t *testing.T) {
body := `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<link rel="alternate" href="https://www.youtube.com/feeds/videos.xml?channel_id=UCxyz789" type="application/rss+xml" title="Channel Name">
</head>
<body></body>
</html>
`
have := FindFeeds(body, base)
youtubeURL := "https://www.youtube.com/feeds/videos.xml?playlist_id="
want := []FeedLink{
{URL: "https://www.youtube.com/feeds/videos.xml?channel_id=UCxyz789", Title: "Channel Name - All"},
{URL: youtubeURL + "UULVxyz789", Title: "Channel Name - Live Streams", TitleOverride: "Channel Name - Live Streams"},
{URL: youtubeURL + "UUSHxyz789", Title: "Channel Name - Short videos", TitleOverride: "Channel Name - Short videos"},
{URL: youtubeURL + "UULFxyz789", Title: "Channel Name - Videos", TitleOverride: "Channel Name - Videos"},
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fatal("invalid result")
}
}
func TestFindFeedsYouTubeNoChannelID(t *testing.T) {
body := `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<link rel="alternate" href="https://www.youtube.com/feeds/videos.xml?channel_id=invalid" type="application/rss+xml" title="Youtube">
</head>
<body></body>
</html>
`
have := FindFeeds(body, base)
want := []FeedLink{
{URL: "https://www.youtube.com/feeds/videos.xml?channel_id=invalid", Title: "Youtube"},
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fatal("invalid result")
}
}
func TestFindFeedsNonYouTubeNoTitleOverride(t *testing.T) {
body := `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<link rel="alternate" href="/blog.xml" type="application/rss+xml" title="Blog">
</head>
<body></body>
</html>
`
have := FindFeeds(body, base)
want := []FeedLink{
{URL: base + "/blog.xml", Title: "Blog"},
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fatal("invalid result")
}
if have[0].TitleOverride != "" {
t.Fatal("expected empty TitleOverride for non-YouTube feed")
}
}
func TestFindIcons(t *testing.T) {
body := `
<!DOCTYPE html>

6
src/frontend/js/app.ts
View File

@@ -459,14 +459,16 @@ export default {
})
}
},
createFeed: function(event) {
var form = event.target
createFeed: function($event) {
var form = $event.target
var data = {
url: form.querySelector('input[name=url]').value,
folder_id: parseInt(form.querySelector('select[name=folder_id]').value) || null,
}
if (this.feedNewChoiceSelected) {
var choice = this.feedNewChoice.find(c => c.url === this.feedNewChoiceSelected)
data.url = this.feedNewChoiceSelected
if (choice && choice.title_override) data.title_override = choice.title_override
}
this.loading.newfeed = true
api.feeds.create(data).then(function(result) {

2
src/frontend/js/templates/index.html
View File

@@ -375,7 +375,7 @@
</button>
<div v-if="settings=='create'">
<p class="cursor-default"><b>{{ $t('new_feed') }}</b></p>
<form action="" @submit.prevent="createFeed(event)" class="mt-4">
<form action="" @submit.prevent="createFeed($event)" class="mt-4">
<label for="feed-url">{{ $t('url') }}</label>
<input id="feed-url" name="url" type="url" class="form-control" required autocomplete="off" :readonly="feedNewChoice.length > 0" placeholder="https://example.com/feed" v-focus>
<label for="feed-folder" class="mt-3 d-block">

5
src/server/forms.go
View File

@@ -16,6 +16,7 @@ type FolderUpdateForm struct {
}
type FeedCreateForm struct {
Url string `json:"url"`
FolderID *int64 `json:"folder_id,omitempty"`
Url string `json:"url"`
TitleOverride string `json:"title_override,omitempty"`
FolderID *int64 `json:"folder_id,omitempty"`
}

6
src/server/routes.go
View File

@@ -268,8 +268,12 @@ func (s *Server) handleFeedList(c *router.Context) {
map[string]any{"status": "multiple", "choice": result.Sources},
)
case result.Feed != nil:
title := result.Feed.Title
if form.TitleOverride != "" {
title = form.TitleOverride
}
feed := s.db.CreateFeed(model.CreateFeedParams{
Title: result.Feed.Title,
Title: title,
Link: result.Feed.SiteURL,
FeedLink: result.FeedLink,
FolderID: form.FolderID,

75
src/server/routes_test.go
View File

@@ -1,6 +1,7 @@
package server
import (
"encoding/json"
"fmt"
"io"
"log"
@@ -8,6 +9,7 @@ import (
"net/http/httptest"
"os"
"reflect"
"strings"
"testing"
"github.com/nkanaev/yarr/src/storage"
@@ -76,6 +78,79 @@ func TestIndexGzipped(t *testing.T) {
}
}
func TestFeedCreateWithTitleOverride(t *testing.T) {
log.SetOutput(io.Discard)
defer log.SetOutput(os.Stderr)
feedSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/rss+xml")
w.Write([]byte(`<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>RSS Title</title>
<link>http://example.com</link>
<item>
<title>Item 1</title>
<link>http://example.com/1</link>
</item>
</channel>
</rss>
`))
}))
defer feedSrv.Close()
db, err := storage.New(":memory:")
if err != nil {
t.Fatal(err)
}
server := NewServer(db, "127.0.0.1:8000")
handler := server.handler()
t.Run("override title", func(t *testing.T) {
body := fmt.Sprintf(`{"url":%q,"title_override":"Override Title"}`, feedSrv.URL)
recorder := httptest.NewRecorder()
request := httptest.NewRequest("POST", "/api/feeds", strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
handler.ServeHTTP(recorder, request)
if recorder.Result().StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Result().StatusCode)
}
var resp map[string]any
json.NewDecoder(recorder.Result().Body).Decode(&resp)
if resp["status"] != "success" {
t.Fatalf("expected success, got %v", resp["status"])
}
feed := resp["feed"].(map[string]any)
if feed["title"] != "Override Title" {
t.Fatalf("expected 'Override Title', got %v", feed["title"])
}
})
t.Run("no override uses rss title", func(t *testing.T) {
body := fmt.Sprintf(`{"url":%q}`, feedSrv.URL)
recorder := httptest.NewRecorder()
request := httptest.NewRequest("POST", "/api/feeds", strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
handler.ServeHTTP(recorder, request)
if recorder.Result().StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Result().StatusCode)
}
var resp map[string]any
json.NewDecoder(recorder.Result().Body).Decode(&resp)
if resp["status"] != "success" {
t.Fatalf("expected success, got %v", resp["status"])
}
feed := resp["feed"].(map[string]any)
if feed["title"] != "RSS Title" {
t.Fatalf("expected 'RSS Title', got %v", feed["title"])
}
})
}
func TestFeedIcons(t *testing.T) {
log.SetOutput(io.Discard)
db, _ := storage.New(":memory:")

16
src/worker/crawler.go
View File

@@ -18,15 +18,10 @@ import (
"golang.org/x/net/html/charset"
)
type FeedSource struct {
Title string `json:"title"`
Url string `json:"url"`
}
type DiscoverResult struct {
Feed *parser.Feed
FeedLink string
Sources []FeedSource
Sources []scraper.FeedLink
}
func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
@@ -64,18 +59,15 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
}
}
}
sources := make([]FeedSource, 0)
for url, title := range scraper.FindFeeds(content, candidateUrl) {
sources = append(sources, FeedSource{Title: title, Url: url})
}
sources := scraper.FindFeeds(content, candidateUrl)
switch {
case len(sources) == 0:
return nil, errors.New("no feeds found at the given url")
case len(sources) == 1:
if sources[0].Url == candidateUrl {
if sources[0].URL == candidateUrl {
return nil, errors.New("recursion")
}
return DiscoverFeed(sources[0].Url)
return DiscoverFeed(sources[0].URL)
}
result.Sources = sources