golines -w src

This commit is contained in:
nkanaev
2026-04-25 22:45:33 +01:00
parent f01c26b2c2
commit f1bdbbc0af
14 changed files with 206 additions and 49 deletions

View File

@@ -27,10 +27,16 @@ var (
blacklistCandidatesRegexp = regexp.MustCompile(`(?i)popupbody|-ad|g-plus`) blacklistCandidatesRegexp = regexp.MustCompile(`(?i)popupbody|-ad|g-plus`)
okMaybeItsACandidateRegexp = regexp.MustCompile(`(?i)and|article|body|column|main|shadow`) okMaybeItsACandidateRegexp = regexp.MustCompile(`(?i)and|article|body|column|main|shadow`)
unlikelyCandidatesRegexp = regexp.MustCompile(`(?i)banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`) unlikelyCandidatesRegexp = regexp.MustCompile(
`(?i)banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`,
)
negativeRegexp = regexp.MustCompile(`(?i)hidden|^hid$|hid$|hid|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby|p-author`) negativeRegexp = regexp.MustCompile(
positiveRegexp = regexp.MustCompile(`(?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story`) `(?i)hidden|^hid$|hid$|hid|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby|p-author`,
)
positiveRegexp = regexp.MustCompile(
`(?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story`,
)
) )
type nodeScores map[*html.Node]float32 type nodeScores map[*html.Node]float32

View File

@@ -146,7 +146,10 @@ func sanitizeAttributes(baseURL, tagName string, attributes []html.Attribute) ([
} }
attrNames = append(attrNames, attribute.Key) attrNames = append(attrNames, attribute.Key)
htmlAttrs = append(htmlAttrs, fmt.Sprintf(`%s="%s"`, attribute.Key, html.EscapeString(value))) htmlAttrs = append(
htmlAttrs,
fmt.Sprintf(`%s="%s"`, attribute.Key, html.EscapeString(value)),
)
} }
extraAttrNames, extraHTMLAttributes := getExtraAttributes(tagName) extraAttrNames, extraHTMLAttributes := getExtraAttributes(tagName)
@@ -161,11 +164,25 @@ func sanitizeAttributes(baseURL, tagName string, attributes []html.Attribute) ([
func getExtraAttributes(tagName string) ([]string, []string) { func getExtraAttributes(tagName string) ([]string, []string) {
switch tagName { switch tagName {
case "a": case "a":
return []string{"rel", "target", "referrerpolicy"}, []string{`rel="noopener noreferrer"`, `target="_blank"`, `referrerpolicy="no-referrer"`} return []string{
"rel",
"target",
"referrerpolicy",
}, []string{
`rel="noopener noreferrer"`,
`target="_blank"`,
`referrerpolicy="no-referrer"`,
}
case "video", "audio": case "video", "audio":
return []string{"controls"}, []string{"controls"} return []string{"controls"}, []string{"controls"}
case "iframe": case "iframe":
return []string{"sandbox", "loading"}, []string{`sandbox="allow-scripts allow-same-origin allow-popups"`, `loading="lazy"`} return []string{
"sandbox",
"loading",
}, []string{
`sandbox="allow-scripts allow-same-origin allow-popups"`,
`loading="lazy"`,
}
case "img": case "img":
return []string{"loading"}, []string{`loading="lazy"`, `referrerpolicy="no-referrer"`} return []string{"loading"}, []string{`loading="lazy"`, `referrerpolicy="no-referrer"`}
default: default:

View File

@@ -91,13 +91,22 @@ func ParseAtom(r io.Reader) (*Feed, error) {
mediaLinks := srcitem.mediaLinks() mediaLinks := srcitem.mediaLinks()
link := firstNonEmpty(srcitem.OrigLink, srcitem.Links.First("alternate"), srcitem.Links.First(""), linkFromID) link := firstNonEmpty(
srcitem.OrigLink,
srcitem.Links.First("alternate"),
srcitem.Links.First(""),
linkFromID,
)
dstfeed.Items = append(dstfeed.Items, Item{ dstfeed.Items = append(dstfeed.Items, Item{
GUID: firstNonEmpty(guidFromID, srcitem.ID, link), GUID: firstNonEmpty(guidFromID, srcitem.ID, link),
Date: dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)), Date: dateParse(firstNonEmpty(srcitem.Published, srcitem.Updated)),
URL: link, URL: link,
Title: srcitem.Title.Text(), Title: srcitem.Title.Text(),
Content: firstNonEmpty(srcitem.Content.String(), srcitem.Summary.String(), srcitem.firstMediaDescription()), Content: firstNonEmpty(
srcitem.Content.String(),
srcitem.Summary.String(),
srcitem.firstMediaDescription(),
),
MediaLinks: mediaLinks, MediaLinks: mediaLinks,
}) })
} }

View File

@@ -40,7 +40,12 @@ func TestSniff(t *testing.T) {
want := testcase.want want := testcase.want
have := sniff(testcase.input) have := sniff(testcase.input)
if want.encoding != have.encoding || want.feedType != have.feedType { if want.encoding != have.encoding || want.feedType != have.feedType {
t.Errorf("Invalid output\n---\n%s\n---\n\nwant=%#v\nhave=%#v", testcase.input, want, have) t.Errorf(
"Invalid output\n---\n%s\n---\n\nwant=%#v\nhave=%#v",
testcase.input,
want,
have,
)
} }
} }
} }

View File

@@ -70,7 +70,10 @@ func (m *media) mediaLinks() []MediaLink {
} else if strings.HasPrefix(content.MediaType, "video/") { } else if strings.HasPrefix(content.MediaType, "video/") {
links = append(links, MediaLink{URL: url, Type: "video", Description: description}) links = append(links, MediaLink{URL: url, Type: "video", Description: description})
} else if content.MediaMedium == "image" || content.MediaMedium == "audio" || content.MediaMedium == "video" { } else if content.MediaMedium == "image" || content.MediaMedium == "audio" || content.MediaMedium == "video" {
links = append(links, MediaLink{URL: url, Type: content.MediaMedium, Description: description}) links = append(
links,
MediaLink{URL: url, Type: content.MediaMedium, Description: description},
)
} else { } else {
if len(content.MediaThumbnails) > 0 { if len(content.MediaThumbnails) > 0 {
links = append(links, MediaLink{ links = append(links, MediaLink{

View File

@@ -42,8 +42,16 @@ func TestRDFFeed(t *testing.T) {
Title: "Mozilla Dot Org", Title: "Mozilla Dot Org",
SiteURL: "http://www.mozilla.org", SiteURL: "http://www.mozilla.org",
Items: []Item{ Items: []Item{
{GUID: "http://www.mozilla.org/status/", URL: "http://www.mozilla.org/status/", Title: "New Status Updates"}, {
{GUID: "http://www.mozilla.org/bugs/", URL: "http://www.mozilla.org/bugs/", Title: "Bugzilla Reorganized"}, GUID: "http://www.mozilla.org/status/",
URL: "http://www.mozilla.org/status/",
Title: "New Status Updates",
},
{
GUID: "http://www.mozilla.org/bugs/",
URL: "http://www.mozilla.org/bugs/",
Title: "Bugzilla Reorganized",
},
}, },
} }

View File

@@ -78,7 +78,8 @@ func ParseRSS(r io.Reader) (*Feed, error) {
for _, e := range srcitem.Enclosures { for _, e := range srcitem.Enclosures {
if strings.HasPrefix(e.Type, "audio/") { if strings.HasPrefix(e.Type, "audio/") {
podcastURL := e.URL podcastURL := e.URL
if srcitem.OrigEnclosureLink != "" && strings.Contains(podcastURL, path.Base(srcitem.OrigEnclosureLink)) { if srcitem.OrigEnclosureLink != "" &&
strings.Contains(podcastURL, path.Base(srcitem.OrigEnclosureLink)) {
podcastURL = srcitem.OrigEnclosureLink podcastURL = srcitem.OrigEnclosureLink
} }
mediaLinks = append(mediaLinks, MediaLink{URL: podcastURL, Type: "audio"}) mediaLinks = append(mediaLinks, MediaLink{URL: podcastURL, Type: "audio"})
@@ -97,11 +98,15 @@ func ParseRSS(r io.Reader) (*Feed, error) {
} }
dstfeed.Items = append(dstfeed.Items, Item{ dstfeed.Items = append(dstfeed.Items, Item{
GUID: firstNonEmpty(srcitem.GUID.GUID, srcitem.Link), GUID: firstNonEmpty(srcitem.GUID.GUID, srcitem.Link),
Date: dateParse(firstNonEmpty(srcitem.DublinCoreDate, srcitem.PubDate)), Date: dateParse(firstNonEmpty(srcitem.DublinCoreDate, srcitem.PubDate)),
URL: firstNonEmpty(srcitem.OrigLink, srcitem.Link, permalink), URL: firstNonEmpty(srcitem.OrigLink, srcitem.Link, permalink),
Title: srcitem.Title, Title: srcitem.Title,
Content: firstNonEmpty(srcitem.ContentEncoded, srcitem.Description, srcitem.firstMediaDescription()), Content: firstNonEmpty(
srcitem.ContentEncoded,
srcitem.Description,
srcitem.firstMediaDescription(),
),
MediaLinks: mediaLinks, MediaLinks: mediaLinks,
}) })
} }

View File

@@ -303,9 +303,21 @@ func TestRSSMultipleMedia(t *testing.T) {
GUID: "http://example.com/posts/1", GUID: "http://example.com/posts/1",
URL: "http://example.com/posts/1", URL: "http://example.com/posts/1",
MediaLinks: []MediaLink{ MediaLinks: []MediaLink{
{URL: "https://example.com/path/to/image1.png", Type: "image", Description: "description 1"}, {
{URL: "https://example.com/path/to/image2.png", Type: "image", Description: "description 2"}, URL: "https://example.com/path/to/image1.png",
{URL: "https://example.com/path/to/video1.mp4", Type: "video", Description: "video description"}, Type: "image",
Description: "description 1",
},
{
URL: "https://example.com/path/to/image2.png",
Type: "image",
Description: "description 2",
},
{
URL: "https://example.com/path/to/video1.mp4",
Type: "video",
Description: "video description",
},
}, },
}, },
} }

View File

@@ -78,7 +78,11 @@ func TestParseFallback(t *testing.T) {
Folders: []Folder{{ Folders: []Folder{{
Title: "foldertitle", Title: "foldertitle",
Feeds: []Feed{ Feeds: []Feed{
{Title: "feedtext", FeedUrl: "https://example.com/feed.xml", SiteUrl: "https://example.com"}, {
Title: "feedtext",
FeedUrl: "https://example.com/feed.xml",
SiteUrl: "https://example.com",
},
}, },
}}, }},
} }

View File

@@ -77,7 +77,8 @@ func (s *Server) handleStatic(c *router.Context) {
c.Out.WriteHeader(http.StatusNotFound) c.Out.WriteHeader(http.StatusNotFound)
return return
} }
http.StripPrefix(s.BasePath+"/static/", http.FileServer(http.FS(assets.FS))).ServeHTTP(c.Out, c.Req) http.StripPrefix(s.BasePath+"/static/", http.FileServer(http.FS(assets.FS))).
ServeHTTP(c.Out, c.Req)
} }
func (s *Server) handleManifest(c *router.Context) { func (s *Server) handleManifest(c *router.Context) {
@@ -236,7 +237,10 @@ func (s *Server) handleFeedList(c *router.Context) {
log.Printf("Faild to discover feed for %s: %s", form.Url, err) log.Printf("Faild to discover feed for %s: %s", form.Url, err)
c.JSON(http.StatusOK, map[string]string{"status": "notfound"}) c.JSON(http.StatusOK, map[string]string{"status": "notfound"})
case len(result.Sources) > 0: case len(result.Sources) > 0:
c.JSON(http.StatusOK, map[string]interface{}{"status": "multiple", "choice": result.Sources}) c.JSON(
http.StatusOK,
map[string]interface{}{"status": "multiple", "choice": result.Sources},
)
case result.Feed != nil: case result.Feed != nil:
feed := s.db.CreateFeed( feed := s.db.CreateFeed(
result.Feed.Title, result.Feed.Title,

View File

@@ -191,7 +191,10 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
terms[idx] = word + "*" terms[idx] = word + "*"
} }
cond = append(cond, "i.search_rowid in (select rowid from search where search match :search)") cond = append(
cond,
"i.search_rowid in (select rowid from search where search match :search)",
)
args = append(args, sql.Named("search", strings.Join(terms, " "))) args = append(args, sql.Named("search", strings.Join(terms, " ")))
} }
if filter.After != nil { if filter.After != nil {
@@ -199,7 +202,13 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
if newestFirst { if newestFirst {
compare = "<" compare = "<"
} }
cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = :after_id)", compare)) cond = append(
cond,
fmt.Sprintf(
"(i.date, i.id) %s (select date, id from items where id = :after_id)",
compare,
),
)
args = append(args, sql.Named("after_id", *filter.After)) args = append(args, sql.Named("after_id", *filter.After))
} }
if filter.IDs != nil && len(*filter.IDs) > 0 { if filter.IDs != nil && len(*filter.IDs) > 0 {
@@ -249,7 +258,12 @@ func (s *Storage) CountItems(filter ItemFilter) int {
return count return count
} }
func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item { func (s *Storage) ListItems(
filter ItemFilter,
limit int,
newestFirst bool,
withContent bool,
) []Item {
predicate, args := listQueryPredicate(filter, newestFirst) predicate, args := listQueryPredicate(filter, newestFirst)
result := make([]Item, 0) result := make([]Item, 0)
@@ -450,7 +464,8 @@ func (s *Storage) DeleteOldItems() {
} }
for feedId, limit := range feedLimits { for feedId, limit := range feedLimits {
result, err := s.db.Exec(` result, err := s.db.Exec(
`
delete from items delete from items
where id in ( where id in (
select i.id select i.id
@@ -463,7 +478,10 @@ func (s *Storage) DeleteOldItems() {
sql.Named("feed_id", feedId), sql.Named("feed_id", feedId),
sql.Named("starred_status", STARRED), sql.Named("starred_status", STARRED),
sql.Named("limit", limit), sql.Named("limit", limit),
sql.Named("date_limit", time.Now().UTC().Add(-time.Hour*time.Duration(24*itemsKeepDays))), sql.Named(
"date_limit",
time.Now().UTC().Add(-time.Hour*time.Duration(24*itemsKeepDays)),
),
) )
if err != nil { if err != nil {
log.Print(err) log.Print(err)

View File

@@ -47,21 +47,62 @@ func testItemsSetup(db *Storage) testItemScope {
db.CreateItems([]Item{ db.CreateItems([]Item{
// feed11 // feed11
{GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)}, {GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)},
{GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)}, // read {
{GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)}, // starred GUID: "item112",
FeedId: feed11.Id,
Title: "title112",
Date: now.Add(time.Hour * 24 * 2),
}, // read
{
GUID: "item113",
FeedId: feed11.Id,
Title: "title113",
Date: now.Add(time.Hour * 24 * 3),
}, // starred
// feed12 // feed12
{GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)}, {GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)},
{GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)}, // read {
GUID: "item122",
FeedId: feed12.Id,
Title: "title122",
Date: now.Add(time.Hour * 24 * 5),
}, // read
// feed21 // feed21
{GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)}, // read {
{GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)}, // starred GUID: "item211",
FeedId: feed21.Id,
Title: "title211",
Date: now.Add(time.Hour * 24 * 6),
}, // read
{
GUID: "item212",
FeedId: feed21.Id,
Title: "title212",
Date: now.Add(time.Hour * 24 * 7),
}, // starred
// feed01 // feed01
{GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)}, {GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)},
{GUID: "item012", FeedId: feed01.Id, Title: "title012", Date: now.Add(time.Hour * 24 * 9)}, // read {
{GUID: "item013", FeedId: feed01.Id, Title: "title013", Date: now.Add(time.Hour * 24 * 10)}, // starred GUID: "item012",
FeedId: feed01.Id,
Title: "title012",
Date: now.Add(time.Hour * 24 * 9),
}, // read
{
GUID: "item013",
FeedId: feed01.Id,
Title: "title013",
Date: now.Add(time.Hour * 24 * 10),
}, // starred
}) })
db.db.Exec(`update items set status = :status where guid in ("item112", "item122", "item211", "item012")`, sql.Named("status", READ)) db.db.Exec(
db.db.Exec(`update items set status = :status where guid in ("item113", "item212", "item013")`, sql.Named("status", STARRED)) `update items set status = :status where guid in ("item112", "item122", "item211", "item012")`,
sql.Named("status", READ),
)
db.db.Exec(
`update items set status = :status where guid in ("item113", "item212", "item013")`,
sql.Named("status", STARRED),
)
return testItemScope{ return testItemScope{
feed11: feed11, feed11: feed11,
@@ -208,7 +249,9 @@ func TestListItemsPaginated(t *testing.T) {
// unread, newest first // unread, newest first
unread := UNREAD unread := UNREAD
have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false)) have = getItemGuids(
db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false),
)
want = []string{"item011", "item121", "item111"} want = []string{"item011", "item121", "item111"}
if !reflect.DeepEqual(have, want) { if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want) t.Logf("want: %#v", want)
@@ -218,7 +261,9 @@ func TestListItemsPaginated(t *testing.T) {
// starred, oldest first // starred, oldest first
starred := STARRED starred := STARRED
have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false)) have = getItemGuids(
db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false),
)
want = []string{"item212", "item013"} want = []string{"item212", "item013"}
if !reflect.DeepEqual(have, want) { if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want) t.Logf("want: %#v", want)

View File

@@ -248,7 +248,11 @@ var wt winTray
// WindowProc callback function that processes messages sent to a window. // WindowProc callback function that processes messages sent to a window.
// https://msdn.microsoft.com/en-us/library/windows/desktop/ms633573(v=vs.85).aspx // https://msdn.microsoft.com/en-us/library/windows/desktop/ms633573(v=vs.85).aspx
func (t *winTray) wndProc(hWnd windows.Handle, message uint32, wParam, lParam uintptr) (lResult uintptr) { func (t *winTray) wndProc(
hWnd windows.Handle,
message uint32,
wParam, lParam uintptr,
) (lResult uintptr) {
const ( const (
WM_RBUTTONUP = 0x0205 WM_RBUTTONUP = 0x0205
WM_LBUTTONUP = 0x0202 WM_LBUTTONUP = 0x0202
@@ -494,7 +498,12 @@ func (t *winTray) convertToSubMenu(menuItemId uint32) (windows.Handle, error) {
return menu, nil return menu, nil
} }
func (t *winTray) addOrUpdateMenuItem(menuItemId uint32, parentId uint32, title string, disabled, checked bool) error { func (t *winTray) addOrUpdateMenuItem(
menuItemId uint32,
parentId uint32,
title string,
disabled, checked bool,
) error {
// https://msdn.microsoft.com/en-us/library/windows/desktop/ms647578(v=vs.85).aspx // https://msdn.microsoft.com/en-us/library/windows/desktop/ms647578(v=vs.85).aspx
const ( const (
MIIM_FTYPE = 0x00000100 MIIM_FTYPE = 0x00000100
@@ -888,7 +897,13 @@ func (item *MenuItem) SetIcon(iconBytes []byte) {
wt.menuItemIcons[uint32(item.id)] = h wt.menuItemIcons[uint32(item.id)] = h
wt.muMenuItemIcons.Unlock() wt.muMenuItemIcons.Unlock()
err = wt.addOrUpdateMenuItem(uint32(item.id), item.parentId(), item.title, item.disabled, item.checked) err = wt.addOrUpdateMenuItem(
uint32(item.id),
item.parentId(),
item.title,
item.disabled,
item.checked,
)
if err != nil { if err != nil {
log.Printf("Unable to addOrUpdateMenuItem: %v", err) log.Printf("Unable to addOrUpdateMenuItem: %v", err)
return return
@@ -905,7 +920,13 @@ func SetTooltip(tooltip string) {
} }
func addOrUpdateMenuItem(item *MenuItem) { func addOrUpdateMenuItem(item *MenuItem) {
err := wt.addOrUpdateMenuItem(uint32(item.id), item.parentId(), item.title, item.disabled, item.checked) err := wt.addOrUpdateMenuItem(
uint32(item.id),
item.parentId(),
item.title,
item.disabled,
item.checked,
)
if err != nil { if err != nil {
log.Printf("Unable to addOrUpdateMenuItem: %v", err) log.Printf("Unable to addOrUpdateMenuItem: %v", err)
return return