james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

code cleanup

Browse Source
This commit is contained in:
Nazar Kanaev 2020-07-24 16:39:12 +01:00
parent c04f54619b
commit b223233318
9 changed files with 112 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
main.go
View File

@ -1,80 +1,10 @@
package main
import (
//"github.com/nkanaev/yarr/storage"
//"github.com/nkanaev/yarr/worker"
"github.com/nkanaev/yarr/server"
//"log"
)
func main() {
/*
store, err := storage.New()
if err != nil {
log.Fatal(err)
}
log.Print(store)
*/
/*
folder := store.CreateFolder("foo")
store.RenameFolder(folder.Id, "bar")
store.ToggleFolderExpanded(folder.Id, false)
log.Print(store.ListFolders())
*/
/*
feed := store.CreateFeed(
"title", "description", "link", "feedlink", "icon", 1)
store.RenameFeed(feed.Id, "newtitle")
log.Print(store.ListFeeds())
*/
/*;
items := make([]storage.Item, 3, 3)
items = append(items, storage.Item{
Id: "id",
FeedId: 0,
Title: "title",
Link: "link",
Description: "description",
Content: "content",
Author: "author",
Date: 1,
DateUpdated: 1,
Status: storage.UNREAD,
Image: "image",
})
items = append(items, storage.Item{
Id: "id2",
FeedId: 0,
Title: "title",
Link: "link",
Description: "description",
Content: "content",
Author: "author",
Date: 1,
DateUpdated: 50,
Status: storage.UNREAD,
Image: "image",
})
items = append(items, storage.Item{
Id: "id",
FeedId: 0,
Title: "title",
Link: "link",
Description: "description",
Content: "content",
Author: "author",
Date: 1,
DateUpdated: 100,
Status: storage.UNREAD,
Image: "image",
})
log.Print(store.CreateItems(items))
log.Print(store.ListItems())
*/
/*
log.Print(worker.FindFeeds("https://horriblesubs.info/"))
log.Print(worker.FindFeeds("http://daringfireball.net/"))
*/
srv := server.New()
srv.ListenAndServe()
}

7
server/crawler.go
View File

@ -1,19 +1,18 @@
package server
import (
"net/url"
"net/http"
"github.com/PuerkitoBio/goquery"
"net/http"
"net/url"
)
type FeedSource struct {
Title string `json:"title"`
Url string `json:"url"`
Url string `json:"url"`
}
const feedLinks = `link[type='application/rss+xml'],link[type='application/atom+xml']`
func FindFeeds(r *http.Response) ([]FeedSource, error) {
sources := make([]FeedSource, 0, 0)
doc, err := goquery.NewDocumentFromResponse(r)

75
server/handlers.go
View File

@ -1,23 +1,23 @@
package server
import (
"github.com/nkanaev/yarr/storage"
"github.com/mmcdole/gofeed"
"net/http"
"html/template"
"encoding/json"
"encoding/xml"
"os"
"log"
"fmt"
"github.com/mmcdole/gofeed"
"github.com/nkanaev/yarr/storage"
"html"
"html/template"
"io"
"io/ioutil"
"log"
"math"
"mime"
"strings"
"net/http"
"os"
"path/filepath"
"strconv"
"math"
"html"
"fmt"
"io/ioutil"
"strings"
)
func IndexHandler(rw http.ResponseWriter, req *http.Request) {
@ -46,7 +46,7 @@ func StaticHandler(rw http.ResponseWriter, req *http.Request) {
func StatusHandler(rw http.ResponseWriter, req *http.Request) {
writeJSON(rw, map[string]interface{}{
"running": handler(req).fetchRunning,
"stats": db(req).FeedStats(),
"stats": db(req).FeedStats(),
})
}
@ -78,10 +78,9 @@ func FolderListHandler(rw http.ResponseWriter, req *http.Request) {
}
}
type UpdateFolder struct {
Title *string `json:"title,omitempty"`
IsExpanded *bool `json:"is_expanded,omitempty"`
Title *string `json:"title,omitempty"`
IsExpanded *bool `json:"is_expanded,omitempty"`
}
func FolderHandler(rw http.ResponseWriter, req *http.Request) {
@ -111,13 +110,13 @@ func FolderHandler(rw http.ResponseWriter, req *http.Request) {
}
type NewFeed struct {
Url string `json:"url"`
Url string `json:"url"`
FolderID *int64 `json:"folder_id,omitempty"`
}
type UpdateFeed struct {
Title *string `json:"title,omitempty"`
FolderID *int64 `json:"folder_id,omitempty"`
Title *string `json:"title,omitempty"`
FolderID *int64 `json:"folder_id,omitempty"`
}
func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
@ -133,7 +132,7 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
}
feedUrl := feed.Url
res, err := http.Get(feedUrl)
res, err := http.Get(feedUrl)
if err != nil {
log.Print(err)
rw.WriteHeader(http.StatusBadRequest)
@ -192,24 +191,24 @@ func convertItems(items []*gofeed.Item, feed storage.Feed) []storage.Item {
author = item.Author.Name
}
result = append(result, storage.Item{
GUID: item.GUID,
FeedId: feed.Id,
Title: item.Title,
Link: item.Link,
GUID: item.GUID,
FeedId: feed.Id,
Title: item.Title,
Link: item.Link,
Description: item.Description,
Content: item.Content,
Author: author,
Date: item.PublishedParsed,
Content: item.Content,
Author: author,
Date: item.PublishedParsed,
DateUpdated: item.UpdatedParsed,
Status: storage.UNREAD,
Image: imageURL,
Status: storage.UNREAD,
Image: imageURL,
})
}
return result
}
func listItems(f storage.Feed) []storage.Item {
fp := gofeed.NewParser()
fp := gofeed.NewParser()
feed, err := fp.ParseURL(f.FeedLink)
if err != nil {
log.Print(err)
@ -365,12 +364,12 @@ type opml struct {
}
type outline struct {
Type string `xml:"type,attr,omitempty"`
Title string `xml:"text,attr"`
FeedURL string `xml:"xmlUrl,attr,omitempty"`
SiteURL string `xml:"htmlUrl,attr,omitempty"`
Description string `xml:"description,attr,omitempty"`
Outlines []outline `xml:"outline,omitempty"`
Type string `xml:"type,attr,omitempty"`
Title string `xml:"text,attr"`
FeedURL string `xml:"xmlUrl,attr,omitempty"`
SiteURL string `xml:"htmlUrl,attr,omitempty"`
Description string `xml:"description,attr,omitempty"`
Outlines []outline `xml:"outline,omitempty"`
}
func (o outline) AllFeeds() []outline {
@ -437,8 +436,8 @@ func OPMLExportHandler(rw http.ResponseWriter, req *http.Request) {
feedline := func(feed storage.Feed, indent int) {
line(
strings.Repeat(" ", indent) +
`<outline type="rss" text="%s" description="%s" xmlUrl="%s" htmlUrl="%s"/>`,
strings.Repeat(" ", indent)+
`<outline type="rss" text="%s" description="%s" xmlUrl="%s" htmlUrl="%s"/>`,
feed.Title, feed.Description,
feed.FeedLink, feed.Link,
)
@ -479,7 +478,7 @@ func OPMLExportHandler(rw http.ResponseWriter, req *http.Request) {
func PageCrawlHandler(rw http.ResponseWriter, req *http.Request) {
query := req.URL.Query()
if url := query.Get("url"); len(url) > 0 {
res, err := http.Get(url)
res, err := http.Get(url)
if err == nil {
body, err := ioutil.ReadAll(res.Body)
if err == nil {

34
server/server.go
View File

@ -1,26 +1,26 @@
package server
import (
"encoding/json"
"context"
"regexp"
"net/http"
"encoding/json"
"github.com/nkanaev/yarr/storage"
"log"
"net/http"
"regexp"
)
type Route struct {
url string
url string
urlRegex *regexp.Regexp
handler func(http.ResponseWriter, *http.Request)
handler func(http.ResponseWriter, *http.Request)
}
type Handler struct {
db *storage.Storage
db *storage.Storage
fetchRunning bool
feedQueue chan storage.Feed
counter chan int
queueSize int
feedQueue chan storage.Feed
counter chan int
queueSize int
}
func (h *Handler) startJobs() {
@ -35,14 +35,14 @@ func (h *Handler) startJobs() {
for {
val := <-h.counter
h.queueSize += val
}
}
}()
go h.db.SyncSearch()
h.fetchAllFeeds()
}
func (h *Handler) fetchFeed(feed storage.Feed) {
h.queueSize += 1
h.queueSize += 1
h.feedQueue <- feed
}
@ -62,9 +62,9 @@ func p(path string, handler func(http.ResponseWriter, *http.Request)) Route {
})
urlRegexp = "^" + urlRegexp + "$"
return Route{
url: path,
url: path,
urlRegex: regexp.MustCompile(urlRegexp),
handler: handler,
handler: handler,
}
}
@ -104,8 +104,8 @@ func handler(req *http.Request) *Handler {
}
const (
ctxDB = 1
ctxVars = 2
ctxDB = 1
ctxVars = 2
ctxHandler = 3
)
@ -143,9 +143,9 @@ func New() *http.Server {
db, _ := storage.New()
db.DeleteOldItems()
h := Handler{
db: db,
db: db,
feedQueue: make(chan storage.Feed),
counter: make(chan int),
counter: make(chan int),
}
s := &http.Server{Addr: "127.0.0.1:8000", Handler: h}
//h.startJobs()

24
storage/feed.go
View File

@ -1,13 +1,13 @@
package storage
type Feed struct {
Id int64 `json:"id"`
FolderId *int64 `json:"folder_id"`
Title string `json:"title"`
Id int64 `json:"id"`
FolderId *int64 `json:"folder_id"`
Title string `json:"title"`
Description string `json:"description"`
Link string `json:"link"`
FeedLink string `json:"feed_link"`
Icon string `json:"icon"`
Link string `json:"link"`
FeedLink string `json:"feed_link"`
Icon string `json:"icon"`
}
func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, folderId *int64) *Feed {
@ -26,13 +26,13 @@ func (s *Storage) CreateFeed(title, description, link, feedLink, icon string, fo
return nil
}
return &Feed{
Id: id,
Title: title,
Id: id,
Title: title,
Description: description,
Link: link,
FeedLink: feedLink,
Icon: icon,
FolderId: folderId,
Link: link,
FeedLink: feedLink,
Icon: icon,
FolderId: folderId,
}
}

6
storage/folder.go
View File

@ -5,9 +5,9 @@ import (
)
type Folder struct {
Id int64 `json:"id"`
Title string `json:"title"`
IsExpanded bool `json:"is_expanded"`
Id int64 `json:"id"`
Title string `json:"title"`
IsExpanded bool `json:"is_expanded"`
}
func (s *Storage) CreateFolder(title string) *Folder {

64
storage/item.go
View File

@ -1,11 +1,11 @@
package storage
import (
"fmt"
"time"
"strings"
"encoding/json"
"fmt"
"golang.org/x/net/html"
"strings"
"time"
)
type ItemStatus int
@ -16,15 +16,15 @@ const (
STARRED ItemStatus = 2
)
var StatusRepresentations = map[ItemStatus]string {
UNREAD: "unread",
READ: "read",
var StatusRepresentations = map[ItemStatus]string{
UNREAD: "unread",
READ: "read",
STARRED: "starred",
}
var StatusValues = map[string]ItemStatus {
"unread": UNREAD,
"read": READ,
var StatusValues = map[string]ItemStatus{
"unread": UNREAD,
"read": READ,
"starred": STARRED,
}
@ -42,25 +42,25 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error {
}
type Item struct {
Id int64 `json:"id"`
GUID string `json:"guid"`
FeedId int64 `json:"feed_id"`
Title string `json:"title"`
Link string `json:"link"`
Description string `json:"description"`
Content string `json:"content"`
Author string `json:"author"`
Date *time.Time `json:"date"`
Id int64 `json:"id"`
GUID string `json:"guid"`
FeedId int64 `json:"feed_id"`
Title string `json:"title"`
Link string `json:"link"`
Description string `json:"description"`
Content string `json:"content"`
Author string `json:"author"`
Date *time.Time `json:"date"`
DateUpdated *time.Time `json:"date_updated"`
Status ItemStatus `json:"status"`
Image string `json:"image"`
Status ItemStatus `json:"status"`
Image string `json:"image"`
}
type ItemFilter struct {
FolderID *int64
FeedID *int64
Status *ItemStatus
Search *string
FeedID *int64
Status *ItemStatus
Search *string
}
func (s *Storage) CreateItems(items []Item) bool {
@ -227,8 +227,8 @@ func (s *Storage) MarkItemsRead(filter ItemFilter) bool {
}
type FeedStat struct {
FeedId int64 `json:"feed_id"`
UnreadCount int64 `json:"unread"`
FeedId int64 `json:"feed_id"`
UnreadCount int64 `json:"unread"`
StarredCount int64 `json:"starred"`
}
@ -255,20 +255,20 @@ func (s *Storage) FeedStats() []FeedStat {
}
func HTMLText(s string) string {
tokenizer := html.NewTokenizer(strings.NewReader(s))
tokenizer := html.NewTokenizer(strings.NewReader(s))
contents := make([]string, 0)
for {
for {
token := tokenizer.Next()
if token == html.ErrorToken {
break
}
if token == html.TextToken {
content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text())))
if len(content) > 0 {
content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text())))
if len(content) > 0 {
contents = append(contents, content)
}
}
}
}
}
return strings.Join(contents, " ")
}
@ -314,7 +314,7 @@ func (s *Storage) SyncSearch() {
func (s *Storage) DeleteOldItems() {
result, err := s.db.Exec(
`delete from items where status = ? and date < ?`,
READ, time.Now().Add(-time.Hour * 24 * 90) /* 90 days */)
READ, time.Now().Add(-time.Hour*24*90) /* 90 days */)
if err != nil {
s.log.Print(err)
return

8
storage/settings.go
View File

@ -4,8 +4,8 @@ import "encoding/json"
func settingsDefaults() map[string]interface{} {
return map[string]interface{}{
"filter": "",
"feed": "",
"filter": "",
"feed": "",
"feed_list_width": 300,
"item_list_width": 300,
}
@ -27,7 +27,7 @@ func (s *Storage) GetSettingsValue(key string) interface{} {
}
func (s *Storage) GetSettings() map[string]interface{} {
result := settingsDefaults()
result := settingsDefaults()
rows, err := s.db.Query(`select key, val from settings;`)
if err != nil {
s.log.Print(err)
@ -66,7 +66,7 @@ func (s *Storage) UpdateSettings(kv map[string]interface{}) bool {
)
if err != nil {
s.log.Print(err)
return false
return false
}
}
return true

8
storage/storage.go
View File

@ -1,10 +1,10 @@
package storage
import (
"os"
"log"
"database/sql"
_ "github.com/mattn/go-sqlite3"
"log"
"os"
)
var initQuery string = `
@ -63,7 +63,7 @@ end;
`
type Storage struct {
db *sql.DB
db *sql.DB
log *log.Logger
}
@ -77,7 +77,7 @@ func New() (*Storage, error) {
if err != nil {
return nil, err
}
logger := log.New(os.Stdout, "storage: ", log.Ldate | log.Ltime | log.Lshortfile)
logger := log.New(os.Stdout, "storage: ", log.Ldate|log.Ltime|log.Lshortfile)
return &Storage{db: db, log: logger}, nil
}