From b22323331874e8bf59f064bddbec529854f08179 Mon Sep 17 00:00:00 2001 From: Nazar Kanaev Date: Fri, 24 Jul 2020 16:39:12 +0100 Subject: [PATCH] code cleanup --- main.go | 70 ------------------------------------------ server/crawler.go | 7 ++--- server/handlers.go | 75 ++++++++++++++++++++++----------------------- server/server.go | 34 ++++++++++---------- storage/feed.go | 24 +++++++-------- storage/folder.go | 6 ++-- storage/item.go | 64 +++++++++++++++++++------------------- storage/settings.go | 8 ++--- storage/storage.go | 8 ++--- 9 files changed, 112 insertions(+), 184 deletions(-) diff --git a/main.go b/main.go index 9a03731..74abe02 100644 --- a/main.go +++ b/main.go @@ -1,80 +1,10 @@ package main import ( - //"github.com/nkanaev/yarr/storage" - //"github.com/nkanaev/yarr/worker" "github.com/nkanaev/yarr/server" - //"log" ) func main() { - /* - store, err := storage.New() - if err != nil { - log.Fatal(err) - } - log.Print(store) - */ - /* - folder := store.CreateFolder("foo") - store.RenameFolder(folder.Id, "bar") - store.ToggleFolderExpanded(folder.Id, false) - log.Print(store.ListFolders()) - */ - /* - feed := store.CreateFeed( - "title", "description", "link", "feedlink", "icon", 1) - store.RenameFeed(feed.Id, "newtitle") - log.Print(store.ListFeeds()) - */ - /*; - items := make([]storage.Item, 3, 3) - items = append(items, storage.Item{ - Id: "id", - FeedId: 0, - Title: "title", - Link: "link", - Description: "description", - Content: "content", - Author: "author", - Date: 1, - DateUpdated: 1, - Status: storage.UNREAD, - Image: "image", - }) - items = append(items, storage.Item{ - Id: "id2", - FeedId: 0, - Title: "title", - Link: "link", - Description: "description", - Content: "content", - Author: "author", - Date: 1, - DateUpdated: 50, - Status: storage.UNREAD, - Image: "image", - }) - items = append(items, storage.Item{ - Id: "id", - FeedId: 0, - Title: "title", - Link: "link", - Description: "description", - Content: "content", - Author: "author", - Date: 1, - DateUpdated: 100, - Status: storage.UNREAD, - Image: "image", - }) - log.Print(store.CreateItems(items)) - log.Print(store.ListItems()) - */ - /* - log.Print(worker.FindFeeds("https://horriblesubs.info/")) - log.Print(worker.FindFeeds("http://daringfireball.net/")) - */ srv := server.New() srv.ListenAndServe() } diff --git a/server/crawler.go b/server/crawler.go index 663da77..99ba9b0 100644 --- a/server/crawler.go +++ b/server/crawler.go @@ -1,19 +1,18 @@ package server import ( - "net/url" - "net/http" "github.com/PuerkitoBio/goquery" + "net/http" + "net/url" ) type FeedSource struct { Title string `json:"title"` - Url string `json:"url"` + Url string `json:"url"` } const feedLinks = `link[type='application/rss+xml'],link[type='application/atom+xml']` - func FindFeeds(r *http.Response) ([]FeedSource, error) { sources := make([]FeedSource, 0, 0) doc, err := goquery.NewDocumentFromResponse(r) diff --git a/server/handlers.go b/server/handlers.go index e131f16..6a375a1 100644 --- a/server/handlers.go +++ b/server/handlers.go @@ -1,23 +1,23 @@ package server import ( - "github.com/nkanaev/yarr/storage" - "github.com/mmcdole/gofeed" - "net/http" - "html/template" "encoding/json" "encoding/xml" - "os" - "log" + "fmt" + "github.com/mmcdole/gofeed" + "github.com/nkanaev/yarr/storage" + "html" + "html/template" "io" + "io/ioutil" + "log" + "math" "mime" - "strings" + "net/http" + "os" "path/filepath" "strconv" - "math" - "html" - "fmt" - "io/ioutil" + "strings" ) func IndexHandler(rw http.ResponseWriter, req *http.Request) { @@ -46,7 +46,7 @@ func StaticHandler(rw http.ResponseWriter, req *http.Request) { func StatusHandler(rw http.ResponseWriter, req *http.Request) { writeJSON(rw, map[string]interface{}{ "running": handler(req).fetchRunning, - "stats": db(req).FeedStats(), + "stats": db(req).FeedStats(), }) } @@ -78,10 +78,9 @@ func FolderListHandler(rw http.ResponseWriter, req *http.Request) { } } - type UpdateFolder struct { - Title *string `json:"title,omitempty"` - IsExpanded *bool `json:"is_expanded,omitempty"` + Title *string `json:"title,omitempty"` + IsExpanded *bool `json:"is_expanded,omitempty"` } func FolderHandler(rw http.ResponseWriter, req *http.Request) { @@ -111,13 +110,13 @@ func FolderHandler(rw http.ResponseWriter, req *http.Request) { } type NewFeed struct { - Url string `json:"url"` + Url string `json:"url"` FolderID *int64 `json:"folder_id,omitempty"` } type UpdateFeed struct { - Title *string `json:"title,omitempty"` - FolderID *int64 `json:"folder_id,omitempty"` + Title *string `json:"title,omitempty"` + FolderID *int64 `json:"folder_id,omitempty"` } func FeedListHandler(rw http.ResponseWriter, req *http.Request) { @@ -133,7 +132,7 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) { } feedUrl := feed.Url - res, err := http.Get(feedUrl) + res, err := http.Get(feedUrl) if err != nil { log.Print(err) rw.WriteHeader(http.StatusBadRequest) @@ -192,24 +191,24 @@ func convertItems(items []*gofeed.Item, feed storage.Feed) []storage.Item { author = item.Author.Name } result = append(result, storage.Item{ - GUID: item.GUID, - FeedId: feed.Id, - Title: item.Title, - Link: item.Link, + GUID: item.GUID, + FeedId: feed.Id, + Title: item.Title, + Link: item.Link, Description: item.Description, - Content: item.Content, - Author: author, - Date: item.PublishedParsed, + Content: item.Content, + Author: author, + Date: item.PublishedParsed, DateUpdated: item.UpdatedParsed, - Status: storage.UNREAD, - Image: imageURL, + Status: storage.UNREAD, + Image: imageURL, }) } return result } func listItems(f storage.Feed) []storage.Item { - fp := gofeed.NewParser() + fp := gofeed.NewParser() feed, err := fp.ParseURL(f.FeedLink) if err != nil { log.Print(err) @@ -365,12 +364,12 @@ type opml struct { } type outline struct { - Type string `xml:"type,attr,omitempty"` - Title string `xml:"text,attr"` - FeedURL string `xml:"xmlUrl,attr,omitempty"` - SiteURL string `xml:"htmlUrl,attr,omitempty"` - Description string `xml:"description,attr,omitempty"` - Outlines []outline `xml:"outline,omitempty"` + Type string `xml:"type,attr,omitempty"` + Title string `xml:"text,attr"` + FeedURL string `xml:"xmlUrl,attr,omitempty"` + SiteURL string `xml:"htmlUrl,attr,omitempty"` + Description string `xml:"description,attr,omitempty"` + Outlines []outline `xml:"outline,omitempty"` } func (o outline) AllFeeds() []outline { @@ -437,8 +436,8 @@ func OPMLExportHandler(rw http.ResponseWriter, req *http.Request) { feedline := func(feed storage.Feed, indent int) { line( - strings.Repeat(" ", indent) + - ``, + strings.Repeat(" ", indent)+ + ``, feed.Title, feed.Description, feed.FeedLink, feed.Link, ) @@ -479,7 +478,7 @@ func OPMLExportHandler(rw http.ResponseWriter, req *http.Request) { func PageCrawlHandler(rw http.ResponseWriter, req *http.Request) { query := req.URL.Query() if url := query.Get("url"); len(url) > 0 { - res, err := http.Get(url) + res, err := http.Get(url) if err == nil { body, err := ioutil.ReadAll(res.Body) if err == nil { diff --git a/server/server.go b/server/server.go index af73bff..1f91e4a 100644 --- a/server/server.go +++ b/server/server.go @@ -1,26 +1,26 @@ package server import ( - "encoding/json" "context" - "regexp" - "net/http" + "encoding/json" "github.com/nkanaev/yarr/storage" "log" + "net/http" + "regexp" ) type Route struct { - url string + url string urlRegex *regexp.Regexp - handler func(http.ResponseWriter, *http.Request) + handler func(http.ResponseWriter, *http.Request) } type Handler struct { - db *storage.Storage + db *storage.Storage fetchRunning bool - feedQueue chan storage.Feed - counter chan int - queueSize int + feedQueue chan storage.Feed + counter chan int + queueSize int } func (h *Handler) startJobs() { @@ -35,14 +35,14 @@ func (h *Handler) startJobs() { for { val := <-h.counter h.queueSize += val - } + } }() go h.db.SyncSearch() h.fetchAllFeeds() } func (h *Handler) fetchFeed(feed storage.Feed) { - h.queueSize += 1 + h.queueSize += 1 h.feedQueue <- feed } @@ -62,9 +62,9 @@ func p(path string, handler func(http.ResponseWriter, *http.Request)) Route { }) urlRegexp = "^" + urlRegexp + "$" return Route{ - url: path, + url: path, urlRegex: regexp.MustCompile(urlRegexp), - handler: handler, + handler: handler, } } @@ -104,8 +104,8 @@ func handler(req *http.Request) *Handler { } const ( - ctxDB = 1 - ctxVars = 2 + ctxDB = 1 + ctxVars = 2 ctxHandler = 3 ) @@ -143,9 +143,9 @@ func New() *http.Server { db, _ := storage.New() db.DeleteOldItems() h := Handler{ - db: db, + db: db, feedQueue: make(chan storage.Feed), - counter: make(chan int), + counter: make(chan int), } s := &http.Server{Addr: "127.0.0.1:8000", Handler: h} //h.startJobs() diff --git a/storage/feed.go b/storage/feed.go index 55fd681..45a6422 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -1,13 +1,13 @@ package storage type Feed struct { - Id int64 `json:"id"` - FolderId *int64 `json:"folder_id"` - Title string `json:"title"` + Id int64 `json:"id"` + FolderId *int64 `json:"folder_id"` + Title string `json:"title"` Description string `json:"description"` - Link string `json:"link"` - FeedLink string `json:"feed_link"` - Icon string `json:"icon"` + Link string `json:"link"` + FeedLink string `json:"feed_link"` + Icon string `json:"icon"` } func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId *int64) *Feed { @@ -26,13 +26,13 @@ func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, fo return nil } return &Feed{ - Id: id, - Title: title, + Id: id, + Title: title, Description: description, - Link: link, - FeedLink: feedLink, - Icon: icon, - FolderId: folderId, + Link: link, + FeedLink: feedLink, + Icon: icon, + FolderId: folderId, } } diff --git a/storage/folder.go b/storage/folder.go index 0a042c4..0aa3886 100644 --- a/storage/folder.go +++ b/storage/folder.go @@ -5,9 +5,9 @@ import ( ) type Folder struct { - Id int64 `json:"id"` - Title string `json:"title"` - IsExpanded bool `json:"is_expanded"` + Id int64 `json:"id"` + Title string `json:"title"` + IsExpanded bool `json:"is_expanded"` } func (s *Storage) CreateFolder(title string) *Folder { diff --git a/storage/item.go b/storage/item.go index 6c11547..5d520a2 100644 --- a/storage/item.go +++ b/storage/item.go @@ -1,11 +1,11 @@ package storage import ( - "fmt" - "time" - "strings" "encoding/json" + "fmt" "golang.org/x/net/html" + "strings" + "time" ) type ItemStatus int @@ -16,15 +16,15 @@ const ( STARRED ItemStatus = 2 ) -var StatusRepresentations = map[ItemStatus]string { - UNREAD: "unread", - READ: "read", +var StatusRepresentations = map[ItemStatus]string{ + UNREAD: "unread", + READ: "read", STARRED: "starred", } -var StatusValues = map[string]ItemStatus { - "unread": UNREAD, - "read": READ, +var StatusValues = map[string]ItemStatus{ + "unread": UNREAD, + "read": READ, "starred": STARRED, } @@ -42,25 +42,25 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error { } type Item struct { - Id int64 `json:"id"` - GUID string `json:"guid"` - FeedId int64 `json:"feed_id"` - Title string `json:"title"` - Link string `json:"link"` - Description string `json:"description"` - Content string `json:"content"` - Author string `json:"author"` - Date *time.Time `json:"date"` + Id int64 `json:"id"` + GUID string `json:"guid"` + FeedId int64 `json:"feed_id"` + Title string `json:"title"` + Link string `json:"link"` + Description string `json:"description"` + Content string `json:"content"` + Author string `json:"author"` + Date *time.Time `json:"date"` DateUpdated *time.Time `json:"date_updated"` - Status ItemStatus `json:"status"` - Image string `json:"image"` + Status ItemStatus `json:"status"` + Image string `json:"image"` } type ItemFilter struct { FolderID *int64 - FeedID *int64 - Status *ItemStatus - Search *string + FeedID *int64 + Status *ItemStatus + Search *string } func (s *Storage) CreateItems(items []Item) bool { @@ -227,8 +227,8 @@ func (s *Storage) MarkItemsRead(filter ItemFilter) bool { } type FeedStat struct { - FeedId int64 `json:"feed_id"` - UnreadCount int64 `json:"unread"` + FeedId int64 `json:"feed_id"` + UnreadCount int64 `json:"unread"` StarredCount int64 `json:"starred"` } @@ -255,20 +255,20 @@ func (s *Storage) FeedStats() []FeedStat { } func HTMLText(s string) string { - tokenizer := html.NewTokenizer(strings.NewReader(s)) + tokenizer := html.NewTokenizer(strings.NewReader(s)) contents := make([]string, 0) - for { + for { token := tokenizer.Next() if token == html.ErrorToken { break } if token == html.TextToken { - content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text()))) - if len(content) > 0 { + content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text()))) + if len(content) > 0 { contents = append(contents, content) - } + } } - } + } return strings.Join(contents, " ") } @@ -314,7 +314,7 @@ func (s *Storage) SyncSearch() { func (s *Storage) DeleteOldItems() { result, err := s.db.Exec( `delete from items where status = ? and date < ?`, - READ, time.Now().Add(-time.Hour * 24 * 90) /* 90 days */) + READ, time.Now().Add(-time.Hour*24*90) /* 90 days */) if err != nil { s.log.Print(err) return diff --git a/storage/settings.go b/storage/settings.go index fccb7b2..7646690 100644 --- a/storage/settings.go +++ b/storage/settings.go @@ -4,8 +4,8 @@ import "encoding/json" func settingsDefaults() map[string]interface{} { return map[string]interface{}{ - "filter": "", - "feed": "", + "filter": "", + "feed": "", "feed_list_width": 300, "item_list_width": 300, } @@ -27,7 +27,7 @@ func (s *Storage) GetSettingsValue(key string) interface{} { } func (s *Storage) GetSettings() map[string]interface{} { - result := settingsDefaults() + result := settingsDefaults() rows, err := s.db.Query(`select key, val from settings;`) if err != nil { s.log.Print(err) @@ -66,7 +66,7 @@ func (s *Storage) UpdateSettings(kv map[string]interface{}) bool { ) if err != nil { s.log.Print(err) - return false + return false } } return true diff --git a/storage/storage.go b/storage/storage.go index dfdbf1e..b788532 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -1,10 +1,10 @@ package storage import ( - "os" - "log" "database/sql" _ "github.com/mattn/go-sqlite3" + "log" + "os" ) var initQuery string = ` @@ -63,7 +63,7 @@ end; ` type Storage struct { - db *sql.DB + db *sql.DB log *log.Logger } @@ -77,7 +77,7 @@ func New() (*Storage, error) { if err != nil { return nil, err } - logger := log.New(os.Stdout, "storage: ", log.Ldate | log.Ltime | log.Lshortfile) + logger := log.New(os.Stdout, "storage: ", log.Ldate|log.Ltime|log.Lshortfile) return &Storage{db: db, log: logger}, nil }