james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

code cleanup

Browse Source
This commit is contained in:
Nazar Kanaev 2020-07-24 16:39:12 +01:00
parent c04f54619b
commit b223233318
9 changed files with 112 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
main.go
View File

@ -1,80 +1,10 @@
package main package main
import ( import (
//"github.com/nkanaev/yarr/storage"
//"github.com/nkanaev/yarr/worker"
"github.com/nkanaev/yarr/server" "github.com/nkanaev/yarr/server"
//"log"
) )
func main() { func main() {
/*
store, err := storage.New()
if err != nil {
log.Fatal(err)
}
log.Print(store)
*/
/*
folder := store.CreateFolder("foo")
store.RenameFolder(folder.Id, "bar")
store.ToggleFolderExpanded(folder.Id, false)
log.Print(store.ListFolders())
*/
/*
feed := store.CreateFeed(
"title", "description", "link", "feedlink", "icon", 1)
store.RenameFeed(feed.Id, "newtitle")
log.Print(store.ListFeeds())
*/
/*;
items := make([]storage.Item, 3, 3)
items = append(items, storage.Item{
Id: "id",
FeedId: 0,
Title: "title",
Link: "link",
Description: "description",
Content: "content",
Author: "author",
Date: 1,
DateUpdated: 1,
Status: storage.UNREAD,
Image: "image",
})
items = append(items, storage.Item{
Id: "id2",
FeedId: 0,
Title: "title",
Link: "link",
Description: "description",
Content: "content",
Author: "author",
Date: 1,
DateUpdated: 50,
Status: storage.UNREAD,
Image: "image",
})
items = append(items, storage.Item{
Id: "id",
FeedId: 0,
Title: "title",
Link: "link",
Description: "description",
Content: "content",
Author: "author",
Date: 1,
DateUpdated: 100,
Status: storage.UNREAD,
Image: "image",
})
log.Print(store.CreateItems(items))
log.Print(store.ListItems())
*/
/*
log.Print(worker.FindFeeds("https://horriblesubs.info/"))
log.Print(worker.FindFeeds("http://daringfireball.net/"))
*/
srv := server.New() srv := server.New()
srv.ListenAndServe() srv.ListenAndServe()
} }

7
server/crawler.go
View File

@ -1,19 +1,18 @@
package server package server
import ( import (
"net/url"
"net/http"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"net/http"
"net/url"
) )
type FeedSource struct { type FeedSource struct {
Title string `json:"title"` Title string `json:"title"`
Url string `json:"url"` Url string `json:"url"`
} }
const feedLinks = `link[type='application/rss+xml'],link[type='application/atom+xml']` const feedLinks = `link[type='application/rss+xml'],link[type='application/atom+xml']`
func FindFeeds(r *http.Response) ([]FeedSource, error) { func FindFeeds(r *http.Response) ([]FeedSource, error) {
sources := make([]FeedSource, 0, 0) sources := make([]FeedSource, 0, 0)
doc, err := goquery.NewDocumentFromResponse(r) doc, err := goquery.NewDocumentFromResponse(r)

75
server/handlers.go
View File

@ -1,23 +1,23 @@
package server package server
import ( import (
"github.com/nkanaev/yarr/storage"
"github.com/mmcdole/gofeed"
"net/http"
"html/template"
"encoding/json" "encoding/json"
"encoding/xml" "encoding/xml"
"os" "fmt"
"log" "github.com/mmcdole/gofeed"
"github.com/nkanaev/yarr/storage"
"html"
"html/template"
"io" "io"
"io/ioutil"
"log"
"math"
"mime" "mime"
"strings" "net/http"
"os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"math" "strings"
"html"
"fmt"
"io/ioutil"
) )
func IndexHandler(rw http.ResponseWriter, req *http.Request) { func IndexHandler(rw http.ResponseWriter, req *http.Request) {
@ -46,7 +46,7 @@ func StaticHandler(rw http.ResponseWriter, req *http.Request) {
func StatusHandler(rw http.ResponseWriter, req *http.Request) { func StatusHandler(rw http.ResponseWriter, req *http.Request) {
writeJSON(rw, map[string]interface{}{ writeJSON(rw, map[string]interface{}{
"running": handler(req).fetchRunning, "running": handler(req).fetchRunning,
"stats": db(req).FeedStats(), "stats": db(req).FeedStats(),
}) })
} }
@ -78,10 +78,9 @@ func FolderListHandler(rw http.ResponseWriter, req *http.Request) {
} }
} }
type UpdateFolder struct { type UpdateFolder struct {
Title *string `json:"title,omitempty"` Title *string `json:"title,omitempty"`
IsExpanded *bool `json:"is_expanded,omitempty"` IsExpanded *bool `json:"is_expanded,omitempty"`
} }
func FolderHandler(rw http.ResponseWriter, req *http.Request) { func FolderHandler(rw http.ResponseWriter, req *http.Request) {
@ -111,13 +110,13 @@ func FolderHandler(rw http.ResponseWriter, req *http.Request) {
} }
type NewFeed struct { type NewFeed struct {
Url string `json:"url"` Url string `json:"url"`
FolderID *int64 `json:"folder_id,omitempty"` FolderID *int64 `json:"folder_id,omitempty"`
} }
type UpdateFeed struct { type UpdateFeed struct {
Title *string `json:"title,omitempty"` Title *string `json:"title,omitempty"`
FolderID *int64 `json:"folder_id,omitempty"` FolderID *int64 `json:"folder_id,omitempty"`
} }
func FeedListHandler(rw http.ResponseWriter, req *http.Request) { func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
@ -133,7 +132,7 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
} }
feedUrl := feed.Url feedUrl := feed.Url
res, err := http.Get(feedUrl) res, err := http.Get(feedUrl)
if err != nil { if err != nil {
log.Print(err) log.Print(err)
rw.WriteHeader(http.StatusBadRequest) rw.WriteHeader(http.StatusBadRequest)
@ -192,24 +191,24 @@ func convertItems(items []*gofeed.Item, feed storage.Feed) []storage.Item {
author = item.Author.Name author = item.Author.Name
} }
result = append(result, storage.Item{ result = append(result, storage.Item{
GUID: item.GUID, GUID: item.GUID,
FeedId: feed.Id, FeedId: feed.Id,
Title: item.Title, Title: item.Title,
Link: item.Link, Link: item.Link,
Description: item.Description, Description: item.Description,
Content: item.Content, Content: item.Content,
Author: author, Author: author,
Date: item.PublishedParsed, Date: item.PublishedParsed,
DateUpdated: item.UpdatedParsed, DateUpdated: item.UpdatedParsed,
Status: storage.UNREAD, Status: storage.UNREAD,
Image: imageURL, Image: imageURL,
}) })
} }
return result return result
} }
func listItems(f storage.Feed) []storage.Item { func listItems(f storage.Feed) []storage.Item {
fp := gofeed.NewParser() fp := gofeed.NewParser()
feed, err := fp.ParseURL(f.FeedLink) feed, err := fp.ParseURL(f.FeedLink)
if err != nil { if err != nil {
log.Print(err) log.Print(err)
@ -365,12 +364,12 @@ type opml struct {
} }
type outline struct { type outline struct {
Type string `xml:"type,attr,omitempty"` Type string `xml:"type,attr,omitempty"`
Title string `xml:"text,attr"` Title string `xml:"text,attr"`
FeedURL string `xml:"xmlUrl,attr,omitempty"` FeedURL string `xml:"xmlUrl,attr,omitempty"`
SiteURL string `xml:"htmlUrl,attr,omitempty"` SiteURL string `xml:"htmlUrl,attr,omitempty"`
Description string `xml:"description,attr,omitempty"` Description string `xml:"description,attr,omitempty"`
Outlines []outline `xml:"outline,omitempty"` Outlines []outline `xml:"outline,omitempty"`
} }
func (o outline) AllFeeds() []outline { func (o outline) AllFeeds() []outline {
@ -437,8 +436,8 @@ func OPMLExportHandler(rw http.ResponseWriter, req *http.Request) {
feedline := func(feed storage.Feed, indent int) { feedline := func(feed storage.Feed, indent int) {
line( line(
strings.Repeat(" ", indent) + strings.Repeat(" ", indent)+
`<outline type="rss" text="%s" description="%s" xmlUrl="%s" htmlUrl="%s"/>`, `<outline type="rss" text="%s" description="%s" xmlUrl="%s" htmlUrl="%s"/>`,
feed.Title, feed.Description, feed.Title, feed.Description,
feed.FeedLink, feed.Link, feed.FeedLink, feed.Link,
) )
@ -479,7 +478,7 @@ func OPMLExportHandler(rw http.ResponseWriter, req *http.Request) {
func PageCrawlHandler(rw http.ResponseWriter, req *http.Request) { func PageCrawlHandler(rw http.ResponseWriter, req *http.Request) {
query := req.URL.Query() query := req.URL.Query()
if url := query.Get("url"); len(url) > 0 { if url := query.Get("url"); len(url) > 0 {
res, err := http.Get(url) res, err := http.Get(url)
if err == nil { if err == nil {
body, err := ioutil.ReadAll(res.Body) body, err := ioutil.ReadAll(res.Body)
if err == nil { if err == nil {

34
server/server.go
View File

@ -1,26 +1,26 @@
package server package server
import ( import (
"encoding/json"
"context" "context"
"regexp" "encoding/json"
"net/http"
"github.com/nkanaev/yarr/storage" "github.com/nkanaev/yarr/storage"
"log" "log"
"net/http"
"regexp"
) )
type Route struct { type Route struct {
url string url string
urlRegex *regexp.Regexp urlRegex *regexp.Regexp
handler func(http.ResponseWriter, *http.Request) handler func(http.ResponseWriter, *http.Request)
} }
type Handler struct { type Handler struct {
db *storage.Storage db *storage.Storage
fetchRunning bool fetchRunning bool
feedQueue chan storage.Feed feedQueue chan storage.Feed
counter chan int counter chan int
queueSize int queueSize int
} }
func (h *Handler) startJobs() { func (h *Handler) startJobs() {
@ -35,14 +35,14 @@ func (h *Handler) startJobs() {
for { for {
val := <-h.counter val := <-h.counter
h.queueSize += val h.queueSize += val
} }
}() }()
go h.db.SyncSearch() go h.db.SyncSearch()
h.fetchAllFeeds() h.fetchAllFeeds()
} }
func (h *Handler) fetchFeed(feed storage.Feed) { func (h *Handler) fetchFeed(feed storage.Feed) {
h.queueSize += 1 h.queueSize += 1
h.feedQueue <- feed h.feedQueue <- feed
} }
@ -62,9 +62,9 @@ func p(path string, handler func(http.ResponseWriter, *http.Request)) Route {
}) })
urlRegexp = "^" + urlRegexp + "$" urlRegexp = "^" + urlRegexp + "$"
return Route{ return Route{
url: path, url: path,
urlRegex: regexp.MustCompile(urlRegexp), urlRegex: regexp.MustCompile(urlRegexp),
handler: handler, handler: handler,
} }
} }
@ -104,8 +104,8 @@ func handler(req *http.Request) *Handler {
} }
const ( const (
ctxDB = 1 ctxDB = 1
ctxVars = 2 ctxVars = 2
ctxHandler = 3 ctxHandler = 3
) )
@ -143,9 +143,9 @@ func New() *http.Server {
db, _ := storage.New() db, _ := storage.New()
db.DeleteOldItems() db.DeleteOldItems()
h := Handler{ h := Handler{
db: db, db: db,
feedQueue: make(chan storage.Feed), feedQueue: make(chan storage.Feed),
counter: make(chan int), counter: make(chan int),
} }
s := &http.Server{Addr: "127.0.0.1:8000", Handler: h} s := &http.Server{Addr: "127.0.0.1:8000", Handler: h}
//h.startJobs() //h.startJobs()

24
storage/feed.go
View File

@ -1,13 +1,13 @@
package storage package storage
type Feed struct { type Feed struct {
Id int64 `json:"id"` Id int64 `json:"id"`
FolderId *int64 `json:"folder_id"` FolderId *int64 `json:"folder_id"`
Title string `json:"title"` Title string `json:"title"`
Description string `json:"description"` Description string `json:"description"`
Link string `json:"link"` Link string `json:"link"`
FeedLink string `json:"feed_link"` FeedLink string `json:"feed_link"`
Icon string `json:"icon"` Icon string `json:"icon"`
} }
func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId *int64) *Feed { func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId *int64) *Feed {
@ -26,13 +26,13 @@ func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, fo
return nil return nil
} }
return &Feed{ return &Feed{
Id: id, Id: id,
Title: title, Title: title,
Description: description, Description: description,
Link: link, Link: link,
FeedLink: feedLink, FeedLink: feedLink,
Icon: icon, Icon: icon,
FolderId: folderId, FolderId: folderId,
} }
} }

6
storage/folder.go
View File

@ -5,9 +5,9 @@ import (
) )
type Folder struct { type Folder struct {
Id int64 `json:"id"` Id int64 `json:"id"`
Title string `json:"title"` Title string `json:"title"`
IsExpanded bool `json:"is_expanded"` IsExpanded bool `json:"is_expanded"`
} }
func (s *Storage) CreateFolder(title string) *Folder { func (s *Storage) CreateFolder(title string) *Folder {

64
storage/item.go
View File

@ -1,11 +1,11 @@
package storage package storage
import ( import (
"fmt"
"time"
"strings"
"encoding/json" "encoding/json"
"fmt"
"golang.org/x/net/html" "golang.org/x/net/html"
"strings"
"time"
) )
type ItemStatus int type ItemStatus int
@ -16,15 +16,15 @@ const (
STARRED ItemStatus = 2 STARRED ItemStatus = 2
) )
var StatusRepresentations = map[ItemStatus]string { var StatusRepresentations = map[ItemStatus]string{
UNREAD: "unread", UNREAD: "unread",
READ: "read", READ: "read",
STARRED: "starred", STARRED: "starred",
} }
var StatusValues = map[string]ItemStatus { var StatusValues = map[string]ItemStatus{
"unread": UNREAD, "unread": UNREAD,
"read": READ, "read": READ,
"starred": STARRED, "starred": STARRED,
} }
@ -42,25 +42,25 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error {
} }
type Item struct { type Item struct {
Id int64 `json:"id"` Id int64 `json:"id"`
GUID string `json:"guid"` GUID string `json:"guid"`
FeedId int64 `json:"feed_id"` FeedId int64 `json:"feed_id"`
Title string `json:"title"` Title string `json:"title"`
Link string `json:"link"` Link string `json:"link"`
Description string `json:"description"` Description string `json:"description"`
Content string `json:"content"` Content string `json:"content"`
Author string `json:"author"` Author string `json:"author"`
Date *time.Time `json:"date"` Date *time.Time `json:"date"`
DateUpdated *time.Time `json:"date_updated"` DateUpdated *time.Time `json:"date_updated"`
Status ItemStatus `json:"status"` Status ItemStatus `json:"status"`
Image string `json:"image"` Image string `json:"image"`
} }
type ItemFilter struct { type ItemFilter struct {
FolderID *int64 FolderID *int64
FeedID *int64 FeedID *int64
Status *ItemStatus Status *ItemStatus
Search *string Search *string
} }
func (s *Storage) CreateItems(items []Item) bool { func (s *Storage) CreateItems(items []Item) bool {
@ -227,8 +227,8 @@ func (s *Storage) MarkItemsRead(filter ItemFilter) bool {
} }
type FeedStat struct { type FeedStat struct {
FeedId int64 `json:"feed_id"` FeedId int64 `json:"feed_id"`
UnreadCount int64 `json:"unread"` UnreadCount int64 `json:"unread"`
StarredCount int64 `json:"starred"` StarredCount int64 `json:"starred"`
} }
@ -255,20 +255,20 @@ func (s *Storage) FeedStats() []FeedStat {
} }
func HTMLText(s string) string { func HTMLText(s string) string {
tokenizer := html.NewTokenizer(strings.NewReader(s)) tokenizer := html.NewTokenizer(strings.NewReader(s))
contents := make([]string, 0) contents := make([]string, 0)
for { for {
token := tokenizer.Next() token := tokenizer.Next()
if token == html.ErrorToken { if token == html.ErrorToken {
break break
} }
if token == html.TextToken { if token == html.TextToken {
content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text()))) content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text())))
if len(content) > 0 { if len(content) > 0 {
contents = append(contents, content) contents = append(contents, content)
} }
} }
} }
return strings.Join(contents, " ") return strings.Join(contents, " ")
} }
@ -314,7 +314,7 @@ func (s *Storage) SyncSearch() {
func (s *Storage) DeleteOldItems() { func (s *Storage) DeleteOldItems() {
result, err := s.db.Exec( result, err := s.db.Exec(
`delete from items where status = ? and date < ?`, `delete from items where status = ? and date < ?`,
READ, time.Now().Add(-time.Hour * 24 * 90) /* 90 days */) READ, time.Now().Add(-time.Hour*24*90) /* 90 days */)
if err != nil { if err != nil {
s.log.Print(err) s.log.Print(err)
return return

8
storage/settings.go
View File

@ -4,8 +4,8 @@ import "encoding/json"
func settingsDefaults() map[string]interface{} { func settingsDefaults() map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"filter": "", "filter": "",
"feed": "", "feed": "",
"feed_list_width": 300, "feed_list_width": 300,
"item_list_width": 300, "item_list_width": 300,
} }
@ -27,7 +27,7 @@ func (s *Storage) GetSettingsValue(key string) interface{} {
} }
func (s *Storage) GetSettings() map[string]interface{} { func (s *Storage) GetSettings() map[string]interface{} {
result := settingsDefaults() result := settingsDefaults()
rows, err := s.db.Query(`select key, val from settings;`) rows, err := s.db.Query(`select key, val from settings;`)
if err != nil { if err != nil {
s.log.Print(err) s.log.Print(err)
@ -66,7 +66,7 @@ func (s *Storage) UpdateSettings(kv map[string]interface{}) bool {
) )
if err != nil { if err != nil {
s.log.Print(err) s.log.Print(err)
return false return false
} }
} }
return true return true

8
storage/storage.go
View File

@ -1,10 +1,10 @@
package storage package storage
import ( import (
"os"
"log"
"database/sql" "database/sql"
_ "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3"
"log"
"os"
) )
var initQuery string = ` var initQuery string = `
@ -63,7 +63,7 @@ end;
` `
type Storage struct { type Storage struct {
db *sql.DB db *sql.DB
log *log.Logger log *log.Logger
} }
@ -77,7 +77,7 @@ func New() (*Storage, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
logger := log.New(os.Stdout, "storage: ", log.Ldate | log.Ltime | log.Lshortfile) logger := log.New(os.Stdout, "storage: ", log.Ldate|log.Ltime|log.Lshortfile)
return &Storage{db: db, log: logger}, nil return &Storage{db: db, log: logger}, nil
} }