james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

fetch favicon

Browse Source
This commit is contained in:
Nazar Kanaev 2020-08-04 15:06:00 +01:00
parent ac5fa62cf3
commit fae9f490cc
7 changed files with 136 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
server/crawler.go
View File

@ -1,7 +1,9 @@
package server
import (
"fmt"
"github.com/PuerkitoBio/goquery"
"io/ioutil"
"net/http"
"net/url"
)
@ -32,3 +34,60 @@ func FindFeeds(r *http.Response) ([]FeedSource, error) {
})
return sources, nil
}
func findFavicon(websiteUrl, feedUrl string) (*[]byte, error) {
candidateUrls := make([]string, 0)
favicon := func(link string) string {
u, err := url.Parse(link)
if err != nil {
return ""
}
return fmt.Sprintf("%s://%s/favicon.ico", u.Scheme, u.Host)
}
if len(websiteUrl) != 0 {
doc, err := goquery.NewDocument(websiteUrl)
if err != nil {
return nil, err
}
doc.Find(`link[rel=icon]`).EachWithBreak(func(i int, s *goquery.Selection) bool {
if href, ok := s.Attr("href"); ok {
if hrefUrl, err := url.Parse(href); err == nil {
faviconUrl := doc.Url.ResolveReference(hrefUrl).String()
candidateUrls = append(candidateUrls, faviconUrl)
}
}
return true
})
if c := favicon(websiteUrl); len(c) != 0 {
candidateUrls = append(candidateUrls, c)
}
}
if c := favicon(feedUrl); len(c) != 0 {
candidateUrls = append(candidateUrls, c)
}
client := http.Client{}
imageTypes := [4]string{
"image/x-icon",
"image/png",
"image/jpeg",
"image/gif",
}
for _, url := range candidateUrls {
if res, err := client.Get(url); err == nil && res.StatusCode == 200 {
if content, err := ioutil.ReadAll(res.Body); err == nil {
ctype := http.DetectContentType(content)
for _, itype := range imageTypes {
if ctype == itype {
return &content, nil
}
}
}
}
}
return nil, nil
}

26
server/handlers.go
View File

@ -27,9 +27,10 @@ var routes []Route = []Route{
p("/api/folders", FolderListHandler),
p("/api/folders/:id", FolderHandler),
p("/api/feeds", FeedListHandler),
p("/api/feeds/refresh", FeedRefreshHandler),
p("/api/feeds/:id", FeedHandler),
p("/api/feeds/find", FeedHandler),
p("/api/feeds/refresh", FeedRefreshHandler),
p("/api/feeds/:id/icon", FeedIconHandler),
p("/api/feeds/:id", FeedHandler),
p("/api/items", ItemListHandler),
p("/api/items/:id", ItemHandler),
p("/api/settings", SettingsHandler),
@ -146,6 +147,22 @@ func FeedRefreshHandler(rw http.ResponseWriter, req *http.Request) {
}
}
func FeedIconHandler(rw http.ResponseWriter, req *http.Request) {
id, err := strconv.ParseInt(Vars(req)["id"], 10, 64)
if err != nil {
rw.WriteHeader(http.StatusBadRequest)
return
}
feed := db(req).GetFeed(id)
if feed != nil && feed.Icon != nil {
rw.Header().Set("Content-Type", http.DetectContentType(*feed.Icon))
rw.Header().Set("Content-Length", strconv.Itoa(len(*feed.Icon)))
rw.Write(*feed.Icon)
} else {
rw.WriteHeader(http.StatusNotFound)
}
}
func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
if req.Method == "GET" {
list := db(req).ListFeeds()
@ -259,7 +276,6 @@ func createFeed(s *storage.Storage, url string, folderId *int64) error {
feed.Description,
feed.Link,
feedLink,
"",
folderId,
)
s.CreateItems(convertItems(feed.Items, *storedFeed))
@ -430,11 +446,11 @@ func OPMLImportHandler(rw http.ResponseWriter, req *http.Request) {
}
for _, outline := range feeds.Outlines {
if outline.Type == "rss" {
db(req).CreateFeed(outline.Title, outline.Description, outline.SiteURL, outline.FeedURL, "", nil)
db(req).CreateFeed(outline.Title, outline.Description, outline.SiteURL, outline.FeedURL, nil)
} else {
folder := db(req).CreateFolder(outline.Title)
for _, o := range outline.AllFeeds() {
db(req).CreateFeed(o.Title, o.Description, o.SiteURL, o.FeedURL, "", &folder.Id)
db(req).CreateFeed(o.Title, o.Description, o.SiteURL, o.FeedURL, &folder.Id)
}
}
}

9
server/server.go
View File

@ -53,6 +53,15 @@ func (h *Handler) startJobs() {
items := listItems(feed)
h.db.CreateItems(items)
atomic.AddInt32(h.queueSize, -1)
if !feed.HasIcon {
icon, err := findFavicon(feed.Link, feed.FeedLink)
if icon != nil {
h.db.UpdateFeedIcon(feed.Id, icon)
}
if err != nil {
h.log.Print(err)
}
}
case <- delTicker.C:
h.db.DeleteOldItems()
}

45
storage/feed.go
View File

@ -11,15 +11,16 @@ type Feed struct {
Description string `json:"description"`
Link string `json:"link"`
FeedLink string `json:"feed_link"`
Icon string `json:"icon"`
Icon *[]byte `json:"icon,omitempty"`
HasIcon bool `json:"has_icon"`
}
func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId *int64) *Feed {
func (s *Storage) CreateFeed(title, description, link, feedLink string, folderId *int64) *Feed {
result, err := s.db.Exec(`
insert into feeds (title, description, link, feed_link, icon, folder_id)
values (?, ?, ?, ?, ?, ?)
insert into feeds (title, description, link, feed_link, folder_id)
values (?, ?, ?, ?, ?)
on conflict (feed_link) do update set folder_id=?`,
html.UnescapeString(title), description, link, feedLink, icon, folderId,
html.UnescapeString(title), description, link, feedLink, folderId,
folderId,
)
if err != nil {
@ -35,7 +36,6 @@ func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, fo
Description: description,
Link: link,
FeedLink: feedLink,
Icon: icon,
FolderId: folderId,
}
}
@ -56,10 +56,16 @@ func (s *Storage) UpdateFeedFolder(feedId int64, newFolderId int64) bool {
return err == nil
}
func (s *Storage) UpdateFeedIcon(feedId int64, icon *[]byte) bool {
_, err := s.db.Exec(`update feeds set icon = ? where id = ?`, icon, feedId)
return err == nil
}
func (s *Storage) ListFeeds() []Feed {
result := make([]Feed, 0, 0)
rows, err := s.db.Query(`
select id, folder_id, title, description, link, feed_link, icon
select id, folder_id, title, description, link, feed_link,
ifnull(icon, '') != '' as has_icon
from feeds
`)
if err != nil {
@ -75,7 +81,7 @@ func (s *Storage) ListFeeds() []Feed {
&f.Description,
&f.Link,
&f.FeedLink,
&f.Icon,
&f.HasIcon,
)
if err != nil {
s.log.Print(err)
@ -85,3 +91,26 @@ func (s *Storage) ListFeeds() []Feed {
}
return result
}
func (s *Storage) GetFeed(id int64) *Feed {
row := s.db.QueryRow(`
select id, folder_id, title, description, link, feed_link, icon,
ifnull(icon, '') != '' as has_icon
from feeds where id = ?
`, id)
if row != nil {
var f Feed
row.Scan(
&f.Id,
&f.FolderId,
&f.Title,
&f.Description,
&f.Link,
&f.FeedLink,
&f.Icon,
&f.HasIcon,
)
return &f
}
return nil
}

2
storage/storage.go
View File

@ -23,7 +23,7 @@ create table if not exists feeds (
description text,
link text,
feed_link text not null,
icon text
icon blob
);
create index if not exists idx_feed_folder_id on feeds(folder_id);

3
template/index.html
View File

@ -114,7 +114,8 @@
v-for="feed in folder.feeds">
<input type="radio" name="feed" :value="'feed:'+feed.id" v-model="feedSelected">
<div class="selectgroup-label d-flex align-items-center w-100">
<span class="icon mr-2">{% inline "rss.svg" %}</span>
<span class="icon mr-2" v-if="!feed.has_icon">{% inline "rss.svg" %}</span>
<span class="icon mr-2" v-else><img :src="'/api/feeds/'+feed.id+'/icon'" alt=""></span>
<span class="flex-fill text-left text-truncate">{{ feed.title }}</span>
<span class="counter text-right">{{filteredFeedStats[feed.id] || ''}}</span>
</div>

2
template/static/stylesheets/app.css
View File

@ -141,7 +141,7 @@ select.form-control:not([multiple]):not([size]) {
line-height: 1;
}
.icon > svg {
.icon > svg , .icon > img {
width: 1rem;
height: 1rem;
vertical-align: top;