fetch favicon

server/crawler.go

@@ -1,7 +1,9 @@
 package server
 
 import (
+	"fmt"
 	"github.com/PuerkitoBio/goquery"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 )
@@ -32,3 +34,60 @@ func FindFeeds(r *http.Response) ([]FeedSource, error) {
 	})
 	return sources, nil
 }
+
+func findFavicon(websiteUrl, feedUrl string) (*[]byte, error) {
+	candidateUrls := make([]string, 0)
+
+	favicon := func(link string) string {
+		u, err := url.Parse(link)
+		if err != nil {
+			return ""
+		}
+		return fmt.Sprintf("%s://%s/favicon.ico", u.Scheme, u.Host)
+	}
+
+	if len(websiteUrl) != 0 {
+		doc, err := goquery.NewDocument(websiteUrl)
+		if err != nil {
+			return nil, err
+		}
+		doc.Find(`link[rel=icon]`).EachWithBreak(func(i int, s *goquery.Selection) bool {
+			if href, ok := s.Attr("href"); ok {
+				if hrefUrl, err := url.Parse(href); err == nil {
+					faviconUrl := doc.Url.ResolveReference(hrefUrl).String()
+					candidateUrls = append(candidateUrls, faviconUrl)
+				}
+			}
+			return true
+		})
+
+		if c := favicon(websiteUrl); len(c) != 0 {
+			candidateUrls = append(candidateUrls, c)
+		}
+	}
+	if c := favicon(feedUrl); len(c) != 0 {
+		candidateUrls = append(candidateUrls, c)
+	}
+
+	client := http.Client{}
+
+	imageTypes := [4]string{
+		"image/x-icon",
+		"image/png",
+		"image/jpeg",
+		"image/gif",
+	}
+	for _, url := range candidateUrls {
+		if res, err := client.Get(url); err == nil && res.StatusCode == 200 {
+			if content, err := ioutil.ReadAll(res.Body); err == nil {
+				ctype := http.DetectContentType(content)
+				for _, itype := range imageTypes {
+					if ctype == itype {
+						return &content, nil
+					}
+				}
+			}
+		}
+	}
+	return nil, nil
+}

server/handlers.go

@@ -27,9 +27,10 @@ var routes []Route = []Route{
 	p("/api/folders", FolderListHandler),
 	p("/api/folders/:id", FolderHandler),
 	p("/api/feeds", FeedListHandler),
-	p("/api/feeds/refresh", FeedRefreshHandler),
-	p("/api/feeds/:id", FeedHandler),
 	p("/api/feeds/find", FeedHandler),
+	p("/api/feeds/refresh", FeedRefreshHandler),
+	p("/api/feeds/:id/icon", FeedIconHandler),
+	p("/api/feeds/:id", FeedHandler),
 	p("/api/items", ItemListHandler),
 	p("/api/items/:id", ItemHandler),
 	p("/api/settings", SettingsHandler),
@@ -146,6 +147,22 @@ func FeedRefreshHandler(rw http.ResponseWriter, req *http.Request) {
 	}
 }
 
+func FeedIconHandler(rw http.ResponseWriter, req *http.Request) {
+	id, err := strconv.ParseInt(Vars(req)["id"], 10, 64)
+	if err != nil {
+		rw.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	feed := db(req).GetFeed(id)
+	if feed != nil && feed.Icon != nil {
+		rw.Header().Set("Content-Type", http.DetectContentType(*feed.Icon))
+		rw.Header().Set("Content-Length", strconv.Itoa(len(*feed.Icon)))
+		rw.Write(*feed.Icon)
+	} else {
+		rw.WriteHeader(http.StatusNotFound)
+	}
+}
+
 func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
 	if req.Method == "GET" {
 		list := db(req).ListFeeds()
@@ -259,7 +276,6 @@ func createFeed(s *storage.Storage, url string, folderId *int64) error {
 		feed.Description,
 		feed.Link,
 		feedLink,
-		"",
 		folderId,
 	)
 	s.CreateItems(convertItems(feed.Items, *storedFeed))
@@ -430,11 +446,11 @@ func OPMLImportHandler(rw http.ResponseWriter, req *http.Request) {
 		}
 		for _, outline := range feeds.Outlines {
 			if outline.Type == "rss" {
-				db(req).CreateFeed(outline.Title, outline.Description, outline.SiteURL, outline.FeedURL, "", nil)
+				db(req).CreateFeed(outline.Title, outline.Description, outline.SiteURL, outline.FeedURL, nil)
 			} else {
 				folder := db(req).CreateFolder(outline.Title)
 				for _, o := range outline.AllFeeds() {
-					db(req).CreateFeed(o.Title, o.Description, o.SiteURL, o.FeedURL, "", &folder.Id)
+					db(req).CreateFeed(o.Title, o.Description, o.SiteURL, o.FeedURL, &folder.Id)
 				}
 			}
 		}

server/server.go

@@ -53,6 +53,15 @@ func (h *Handler) startJobs() {
 				items := listItems(feed)
 				h.db.CreateItems(items)
 				atomic.AddInt32(h.queueSize, -1)
+				if !feed.HasIcon {
+					icon, err := findFavicon(feed.Link, feed.FeedLink)
+					if icon != nil {
+						h.db.UpdateFeedIcon(feed.Id, icon)
+					}
+					if err != nil {
+						h.log.Print(err)
+					}
+				}
 			case <- delTicker.C:
 				h.db.DeleteOldItems()
 			}