fetch favicon

server/crawler.go

@@ -1,7 +1,9 @@
 package server
 
 import (
+	"fmt"
 	"github.com/PuerkitoBio/goquery"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 )
@@ -32,3 +34,60 @@ func FindFeeds(r *http.Response) ([]FeedSource, error) {
 	})
 	return sources, nil
 }
+
+func findFavicon(websiteUrl, feedUrl string) (*[]byte, error) {
+	candidateUrls := make([]string, 0)
+
+	favicon := func(link string) string {
+		u, err := url.Parse(link)
+		if err != nil {
+			return ""
+		}
+		return fmt.Sprintf("%s://%s/favicon.ico", u.Scheme, u.Host)
+	}
+
+	if len(websiteUrl) != 0 {
+		doc, err := goquery.NewDocument(websiteUrl)
+		if err != nil {
+			return nil, err
+		}
+		doc.Find(`link[rel=icon]`).EachWithBreak(func(i int, s *goquery.Selection) bool {
+			if href, ok := s.Attr("href"); ok {
+				if hrefUrl, err := url.Parse(href); err == nil {
+					faviconUrl := doc.Url.ResolveReference(hrefUrl).String()
+					candidateUrls = append(candidateUrls, faviconUrl)
+				}
+			}
+			return true
+		})
+
+		if c := favicon(websiteUrl); len(c) != 0 {
+			candidateUrls = append(candidateUrls, c)
+		}
+	}
+	if c := favicon(feedUrl); len(c) != 0 {
+		candidateUrls = append(candidateUrls, c)
+	}
+
+	client := http.Client{}
+
+	imageTypes := [4]string{
+		"image/x-icon",
+		"image/png",
+		"image/jpeg",
+		"image/gif",
+	}
+	for _, url := range candidateUrls {
+		if res, err := client.Get(url); err == nil && res.StatusCode == 200 {
+			if content, err := ioutil.ReadAll(res.Body); err == nil {
+				ctype := http.DetectContentType(content)
+				for _, itype := range imageTypes {
+					if ctype == itype {
+						return &content, nil
+					}
+				}
+			}
+		}
+	}
+	return nil, nil
+}

server/handlers.go

@@ -27,9 +27,10 @@ var routes []Route = []Route{
 	p("/api/folders", FolderListHandler),
 	p("/api/folders/:id", FolderHandler),
 	p("/api/feeds", FeedListHandler),
-	p("/api/feeds/refresh", FeedRefreshHandler),
-	p("/api/feeds/:id", FeedHandler),
 	p("/api/feeds/find", FeedHandler),
+	p("/api/feeds/refresh", FeedRefreshHandler),
+	p("/api/feeds/:id/icon", FeedIconHandler),
+	p("/api/feeds/:id", FeedHandler),
 	p("/api/items", ItemListHandler),
 	p("/api/items/:id", ItemHandler),
 	p("/api/settings", SettingsHandler),
@@ -146,6 +147,22 @@ func FeedRefreshHandler(rw http.ResponseWriter, req *http.Request) {
 	}
 }
 
+func FeedIconHandler(rw http.ResponseWriter, req *http.Request) {
+	id, err := strconv.ParseInt(Vars(req)["id"], 10, 64)
+	if err != nil {
+		rw.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	feed := db(req).GetFeed(id)
+	if feed != nil && feed.Icon != nil {
+		rw.Header().Set("Content-Type", http.DetectContentType(*feed.Icon))
+		rw.Header().Set("Content-Length", strconv.Itoa(len(*feed.Icon)))
+		rw.Write(*feed.Icon)
+	} else {
+		rw.WriteHeader(http.StatusNotFound)
+	}
+}
+
 func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
 	if req.Method == "GET" {
 		list := db(req).ListFeeds()
@@ -259,7 +276,6 @@ func createFeed(s *storage.Storage, url string, folderId *int64) error {
 		feed.Description,
 		feed.Link,
 		feedLink,
-		"",
 		folderId,
 	)
 	s.CreateItems(convertItems(feed.Items, *storedFeed))
@@ -430,11 +446,11 @@ func OPMLImportHandler(rw http.ResponseWriter, req *http.Request) {
 		}
 		for _, outline := range feeds.Outlines {
 			if outline.Type == "rss" {
-				db(req).CreateFeed(outline.Title, outline.Description, outline.SiteURL, outline.FeedURL, "", nil)
+				db(req).CreateFeed(outline.Title, outline.Description, outline.SiteURL, outline.FeedURL, nil)
 			} else {
 				folder := db(req).CreateFolder(outline.Title)
 				for _, o := range outline.AllFeeds() {
-					db(req).CreateFeed(o.Title, o.Description, o.SiteURL, o.FeedURL, "", &folder.Id)
+					db(req).CreateFeed(o.Title, o.Description, o.SiteURL, o.FeedURL, &folder.Id)
 				}
 			}
 		}

server/server.go

@@ -53,6 +53,15 @@ func (h *Handler) startJobs() {
 				items := listItems(feed)
 				h.db.CreateItems(items)
 				atomic.AddInt32(h.queueSize, -1)
+				if !feed.HasIcon {
+					icon, err := findFavicon(feed.Link, feed.FeedLink)
+					if icon != nil {
+						h.db.UpdateFeedIcon(feed.Id, icon)
+					}
+					if err != nil {
+						h.log.Print(err)
+					}
+				}
 			case <- delTicker.C:
 				h.db.DeleteOldItems()
 			}

storage/feed.go

@@ -11,15 +11,16 @@ type Feed struct {
 	Description string  `json:"description"`
 	Link        string  `json:"link"`
 	FeedLink    string  `json:"feed_link"`
-	Icon        string `json:"icon"`
+	Icon        *[]byte `json:"icon,omitempty"`
+	HasIcon     bool    `json:"has_icon"`
 }
 
-func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId *int64) *Feed {
+func (s *Storage) CreateFeed(title, description, link, feedLink string, folderId *int64) *Feed {
 	result, err := s.db.Exec(`
-		insert into feeds (title, description, link, feed_link, icon, folder_id) 
+		insert into feeds (title, description, link, feed_link, folder_id) 
-		values (?, ?, ?, ?, ?, ?)
+		values (?, ?, ?, ?, ?)
 		on conflict (feed_link) do update set folder_id=?`,
-		html.UnescapeString(title), description, link, feedLink, icon, folderId,
+		html.UnescapeString(title), description, link, feedLink, folderId,
 		folderId,
 	)
 	if err != nil {
@@ -35,7 +36,6 @@ func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, fo
 		Description: description,
 		Link:        link,
 		FeedLink:    feedLink,
-		Icon:        icon,
 		FolderId:    folderId,
 	}
 }
@@ -56,10 +56,16 @@ func (s *Storage) UpdateFeedFolder(feedId int64, newFolderId int64) bool {
 	return err == nil
 }
 
+func (s *Storage) UpdateFeedIcon(feedId int64, icon *[]byte) bool {
+	_, err := s.db.Exec(`update feeds set icon = ? where id = ?`, icon, feedId)
+	return err == nil
+}
+
 func (s *Storage) ListFeeds() []Feed {
 	result := make([]Feed, 0, 0)
 	rows, err := s.db.Query(`
-		select id, folder_id, title, description, link, feed_link, icon
+		select id, folder_id, title, description, link, feed_link,
+		       ifnull(icon, '') != '' as has_icon
 		from feeds
 	`)
 	if err != nil {
@@ -75,7 +81,7 @@ func (s *Storage) ListFeeds() []Feed {
 			&f.Description,
 			&f.Link,
 			&f.FeedLink,
-			&f.Icon,
+			&f.HasIcon,
 		)
 		if err != nil {
 			s.log.Print(err)
@@ -85,3 +91,26 @@ func (s *Storage) ListFeeds() []Feed {
 	}
 	return result
 }
+
+func (s *Storage) GetFeed(id int64) *Feed {
+	row := s.db.QueryRow(`
+		select id, folder_id, title, description, link, feed_link, icon,
+		       ifnull(icon, '') != '' as has_icon
+		from feeds where id = ?
+	`, id)
+	if row != nil {
+		var f Feed
+		row.Scan(
+			&f.Id,
+			&f.FolderId,
+			&f.Title,
+			&f.Description,
+			&f.Link,
+			&f.FeedLink,
+			&f.Icon,
+			&f.HasIcon,
+		)
+		return &f
+	}
+	return nil
+}

storage/storage.go

@@ -23,7 +23,7 @@ create table if not exists feeds (
  description    text,
  link           text,
  feed_link      text not null,
- icon           text
+ icon           blob
 );
 
 create index if not exists idx_feed_folder_id on feeds(folder_id);

template/index.html

@@ -114,7 +114,8 @@
                                v-for="feed in folder.feeds">
                             <input type="radio" name="feed" :value="'feed:'+feed.id" v-model="feedSelected">
                             <div class="selectgroup-label d-flex align-items-center w-100">
-                                <span class="icon mr-2">{% inline "rss.svg" %}</span>
+                                <span class="icon mr-2" v-if="!feed.has_icon">{% inline "rss.svg" %}</span>
+                                <span class="icon mr-2" v-else><img :src="'/api/feeds/'+feed.id+'/icon'" alt=""></span>
                                 <span class="flex-fill text-left text-truncate">{{ feed.title }}</span>
                                 <span class="counter text-right">{{filteredFeedStats[feed.id] || ''}}</span>
                             </div>

template/static/stylesheets/app.css

@@ -141,7 +141,7 @@ select.form-control:not([multiple]):not([size]) {
   line-height: 1;
 }
 
-.icon > svg {
+.icon > svg , .icon > img {
   width: 1rem;
   height: 1rem;
   vertical-align: top;