move packages to src

This commit is contained in:
Nazar Kanaev
2021-02-26 13:39:30 +00:00
parent d825ce9bdf
commit 3fac9bb1bd
71 changed files with 0 additions and 0 deletions

172
src/storage/feed.go Normal file
View File

@@ -0,0 +1,172 @@
package storage
import (
"html"
"net/url"
)
type Feed struct {
Id int64 `json:"id"`
FolderId *int64 `json:"folder_id"`
Title string `json:"title"`
Description string `json:"description"`
Link string `json:"link"`
FeedLink string `json:"feed_link"`
Icon *[]byte `json:"icon,omitempty"`
HasIcon bool `json:"has_icon"`
}
func (s *Storage) CreateFeed(title, description, link, feedLink string, folderId *int64) *Feed {
title = html.UnescapeString(title)
// WILD: fallback to `feed.link` -> `feed.feed_link` -> "<???>" if title is missing
if title == "" {
title = link
// use domain if possible
linkUrl, err := url.Parse(link)
if err == nil && linkUrl.Host != "" && len(linkUrl.Path) <= 1 {
title = linkUrl.Host
}
}
if title == "" {
title = feedLink
}
if title == "" {
title = "<???>"
}
result, err := s.db.Exec(`
insert into feeds (title, description, link, feed_link, folder_id)
values (?, ?, ?, ?, ?)
on conflict (feed_link) do update set folder_id=?`,
title, description, link, feedLink, folderId,
folderId,
)
if err != nil {
return nil
}
id, idErr := result.LastInsertId()
if idErr != nil {
return nil
}
return &Feed{
Id: id,
Title: title,
Description: description,
Link: link,
FeedLink: feedLink,
FolderId: folderId,
}
}
func (s *Storage) DeleteFeed(feedId int64) bool {
_, err1 := s.db.Exec(`delete from items where feed_id = ?`, feedId)
_, err2 := s.db.Exec(`delete from feeds where id = ?`, feedId)
return err1 == nil && err2 == nil
}
func (s *Storage) RenameFeed(feedId int64, newTitle string) bool {
_, err := s.db.Exec(`update feeds set title = ? where id = ?`, newTitle, feedId)
return err == nil
}
func (s *Storage) UpdateFeedFolder(feedId int64, newFolderId *int64) bool {
_, err := s.db.Exec(`update feeds set folder_id = ? where id = ?`, newFolderId, feedId)
return err == nil
}
func (s *Storage) UpdateFeedIcon(feedId int64, icon *[]byte) bool {
_, err := s.db.Exec(`update feeds set icon = ? where id = ?`, icon, feedId)
return err == nil
}
func (s *Storage) ListFeeds() []Feed {
result := make([]Feed, 0, 0)
rows, err := s.db.Query(`
select id, folder_id, title, description, link, feed_link,
ifnull(icon, '') != '' as has_icon
from feeds
order by title collate nocase
`)
if err != nil {
s.log.Print(err)
return result
}
for rows.Next() {
var f Feed
err = rows.Scan(
&f.Id,
&f.FolderId,
&f.Title,
&f.Description,
&f.Link,
&f.FeedLink,
&f.HasIcon,
)
if err != nil {
s.log.Print(err)
return result
}
result = append(result, f)
}
return result
}
func (s *Storage) GetFeed(id int64) *Feed {
row := s.db.QueryRow(`
select id, folder_id, title, description, link, feed_link, icon,
ifnull(icon, '') != '' as has_icon
from feeds where id = ?
`, id)
if row != nil {
var f Feed
row.Scan(
&f.Id,
&f.FolderId,
&f.Title,
&f.Description,
&f.Link,
&f.FeedLink,
&f.Icon,
&f.HasIcon,
)
return &f
}
return nil
}
func (s *Storage) ResetFeedErrors() {
if _, err := s.db.Exec(`delete from feed_errors`); err != nil {
s.log.Print(err)
}
}
func (s *Storage) SetFeedError(feedID int64, lastError error) {
_, err := s.db.Exec(`
insert into feed_errors (feed_id, error)
values (?, ?)
on conflict (feed_id) do update set error = excluded.error`,
feedID, lastError.Error(),
)
if err != nil {
s.log.Print(err)
}
}
func (s *Storage) GetFeedErrors() map[int64]string {
errors := make(map[int64]string)
rows, err := s.db.Query(`select feed_id, error from feed_errors`)
if err != nil {
s.log.Print(err)
return errors
}
for rows.Next() {
var id int64
var error string
if err = rows.Scan(&id, &error); err != nil {
s.log.Print(err)
}
errors[id] = error
}
return errors
}

84
src/storage/folder.go Normal file
View File

@@ -0,0 +1,84 @@
package storage
import (
"fmt"
)
type Folder struct {
Id int64 `json:"id"`
Title string `json:"title"`
IsExpanded bool `json:"is_expanded"`
}
func (s *Storage) CreateFolder(title string) *Folder {
expanded := true
result, err := s.db.Exec(`
insert into folders (title, is_expanded) values (?, ?)
on conflict (title) do nothing`,
title, expanded,
)
if err != nil {
fmt.Println(err)
return nil
}
var id int64
numrows, err := result.RowsAffected()
if err != nil {
s.log.Print(err)
return nil
}
if numrows == 1 {
id, err = result.LastInsertId()
if err != nil {
s.log.Print(err)
return nil
}
} else {
err = s.db.QueryRow(`select id, is_expanded from folders where title=?`, title).Scan(&id, &expanded)
if err != nil {
s.log.Print(err)
return nil
}
}
return &Folder{Id: id, Title: title, IsExpanded: expanded}
}
func (s *Storage) DeleteFolder(folderId int64) bool {
_, err1 := s.db.Exec(`update feeds set folder_id = null where folder_id = ?`, folderId)
_, err2 := s.db.Exec(`delete from folders where id = ?`, folderId)
return err1 == nil && err2 == nil
}
func (s *Storage) RenameFolder(folderId int64, newTitle string) bool {
_, err := s.db.Exec(`update folders set title = ? where id = ?`, newTitle, folderId)
return err == nil
}
func (s *Storage) ToggleFolderExpanded(folderId int64, isExpanded bool) bool {
_, err := s.db.Exec(`update folders set is_expanded = ? where id = ?`, isExpanded, folderId)
return err == nil
}
func (s *Storage) ListFolders() []Folder {
result := make([]Folder, 0, 0)
rows, err := s.db.Query(`
select id, title, is_expanded
from folders
order by title collate nocase
`)
if err != nil {
s.log.Print(err)
return result
}
for rows.Next() {
var f Folder
err = rows.Scan(&f.Id, &f.Title, &f.IsExpanded)
if err != nil {
s.log.Print(err)
return result
}
result = append(result, f)
}
return result
}

72
src/storage/http.go Normal file
View File

@@ -0,0 +1,72 @@
package storage
import (
"time"
)
type HTTPState struct {
FeedID int64
LastRefreshed time.Time
LastModified string
Etag string
}
func (s *Storage) ListHTTPStates() map[int64]HTTPState {
result := make(map[int64]HTTPState)
rows, err := s.db.Query(`select feed_id, last_refreshed, last_modified, etag from http_states`)
if err != nil {
s.log.Print(err)
return result
}
for rows.Next() {
var state HTTPState
err = rows.Scan(
&state.FeedID,
&state.LastRefreshed,
&state.LastModified,
&state.Etag,
)
if err != nil {
s.log.Print(err)
return result
}
result[state.FeedID] = state
}
return result
}
func (s *Storage) GetHTTPState(feedID int64) *HTTPState {
row := s.db.QueryRow(`
select feed_id, last_refreshed, last_modified, etag
from http_states where feed_id = ?
`, feedID)
if row == nil {
return nil
}
var state HTTPState
row.Scan(
&state.FeedID,
&state.LastRefreshed,
&state.LastModified,
&state.Etag,
)
return &state
}
func (s *Storage) SetHTTPState(feedID int64, lastModified, etag string) {
_, err := s.db.Exec(`
insert into http_states (feed_id, last_modified, etag, last_refreshed)
values (?, ?, ?, datetime())
on conflict (feed_id) do update set last_modified = ?, etag = ?, last_refreshed = datetime()`,
// insert
feedID, lastModified, etag,
// upsert
lastModified, etag,
)
if err != nil {
s.log.Print(err)
}
}

382
src/storage/item.go Normal file
View File

@@ -0,0 +1,382 @@
package storage
import (
"encoding/json"
"fmt"
"html"
"strings"
"time"
xhtml "golang.org/x/net/html"
)
type ItemStatus int
const (
UNREAD ItemStatus = 0
READ ItemStatus = 1
STARRED ItemStatus = 2
)
var StatusRepresentations = map[ItemStatus]string{
UNREAD: "unread",
READ: "read",
STARRED: "starred",
}
var StatusValues = map[string]ItemStatus{
"unread": UNREAD,
"read": READ,
"starred": STARRED,
}
func (s ItemStatus) MarshalJSON() ([]byte, error) {
return json.Marshal(StatusRepresentations[s])
}
func (s *ItemStatus) UnmarshalJSON(b []byte) error {
var str string
if err := json.Unmarshal(b, &str); err != nil {
return err
}
*s = StatusValues[str]
return nil
}
type Item struct {
Id int64 `json:"id"`
GUID string `json:"guid"`
FeedId int64 `json:"feed_id"`
Title string `json:"title"`
Link string `json:"link"`
Description string `json:"description"`
Content string `json:"content"`
Author string `json:"author"`
Date *time.Time `json:"date"`
DateUpdated *time.Time `json:"date_updated"`
Status ItemStatus `json:"status"`
Image string `json:"image"`
}
type ItemFilter struct {
FolderID *int64
FeedID *int64
Status *ItemStatus
Search *string
}
type MarkFilter struct {
FolderID *int64
FeedID *int64
}
func (s *Storage) CreateItems(items []Item) bool {
tx, err := s.db.Begin()
if err != nil {
s.log.Print(err)
return false
}
now := time.Now()
for _, item := range items {
// WILD: some feeds provide only `item.date_updated` (without `item.date_created`)
if item.Date == nil {
item.Date = item.DateUpdated
}
// WILD: `item.guid` is not always present
if item.GUID == "" {
item.GUID = item.Link
}
_, err = tx.Exec(`
insert into items (
guid, feed_id, title, link, description,
content, author,
date, date_updated, date_arrived,
status, image
)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict (feed_id, guid) do update set
date_updated = ?, date_arrived = ?`,
item.GUID, item.FeedId, html.UnescapeString(item.Title), item.Link, item.Description,
item.Content, item.Author,
item.Date, item.DateUpdated, now,
UNREAD, item.Image,
// upsert values
item.DateUpdated, now,
)
if err != nil {
s.log.Print(err)
if err = tx.Rollback(); err != nil {
s.log.Print(err)
return false
}
return false
}
}
if err = tx.Commit(); err != nil {
s.log.Print(err)
return false
}
return true
}
func listQueryPredicate(filter ItemFilter) (string, []interface{}) {
cond := make([]string, 0)
args := make([]interface{}, 0)
if filter.FolderID != nil {
cond = append(cond, "f.folder_id = ?")
args = append(args, *filter.FolderID)
}
if filter.FeedID != nil {
cond = append(cond, "i.feed_id = ?")
args = append(args, *filter.FeedID)
}
if filter.Status != nil {
cond = append(cond, "i.status = ?")
args = append(args, *filter.Status)
}
if filter.Search != nil {
words := strings.Fields(*filter.Search)
terms := make([]string, len(words))
for idx, word := range words {
terms[idx] = word + "*"
}
cond = append(cond, "i.search_rowid in (select rowid from search where search match ?)")
args = append(args, strings.Join(terms, " "))
}
predicate := "1"
if len(cond) > 0 {
predicate = strings.Join(cond, " and ")
}
return predicate, args
}
func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bool) []Item {
predicate, args := listQueryPredicate(filter)
result := make([]Item, 0, 0)
order := "date desc"
if !newestFirst {
order = "date asc"
}
query := fmt.Sprintf(`
select
i.id, i.guid, i.feed_id, i.title, i.link, i.description,
i.content, i.author, i.date, i.date_updated, i.status, i.image
from items i
join feeds f on f.id = i.feed_id
where %s
order by %s
limit %d offset %d
`, predicate, order, limit, offset)
rows, err := s.db.Query(query, args...)
if err != nil {
s.log.Print(err)
return result
}
for rows.Next() {
var x Item
err = rows.Scan(
&x.Id,
&x.GUID,
&x.FeedId,
&x.Title,
&x.Link,
&x.Description,
&x.Content,
&x.Author,
&x.Date,
&x.DateUpdated,
&x.Status,
&x.Image,
)
if err != nil {
s.log.Print(err)
return result
}
result = append(result, x)
}
return result
}
func (s *Storage) CountItems(filter ItemFilter) int64 {
predicate, args := listQueryPredicate(filter)
query := fmt.Sprintf(`
select count(i.id)
from items i
join feeds f on f.id = i.feed_id
where %s`, predicate)
row := s.db.QueryRow(query, args...)
if row != nil {
var result int64
row.Scan(&result)
return result
}
return 0
}
func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
_, err := s.db.Exec(`update items set status = ? where id = ?`, status, item_id)
return err == nil
}
func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
cond := make([]string, 0)
args := make([]interface{}, 0)
if filter.FolderID != nil {
cond = append(cond, "f.folder_id = ?")
args = append(args, *filter.FolderID)
}
if filter.FeedID != nil {
cond = append(cond, "i.feed_id = ?")
args = append(args, *filter.FeedID)
}
predicate := "1"
if len(cond) > 0 {
predicate = strings.Join(cond, " and ")
}
query := fmt.Sprintf(`
update items set status = %d
where id in (
select i.id from items i
join feeds f on f.id = i.feed_id
where %s and i.status != %d
)
`, READ, predicate, STARRED)
_, err := s.db.Exec(query, args...)
if err != nil {
s.log.Print(err)
}
return err == nil
}
type FeedStat struct {
FeedId int64 `json:"feed_id"`
UnreadCount int64 `json:"unread"`
StarredCount int64 `json:"starred"`
}
func (s *Storage) FeedStats() []FeedStat {
result := make([]FeedStat, 0)
rows, err := s.db.Query(fmt.Sprintf(`
select
feed_id,
sum(case status when %d then 1 else 0 end),
sum(case status when %d then 1 else 0 end)
from items
group by feed_id
`, UNREAD, STARRED))
if err != nil {
s.log.Print(err)
return result
}
for rows.Next() {
stat := FeedStat{}
rows.Scan(&stat.FeedId, &stat.UnreadCount, &stat.StarredCount)
result = append(result, stat)
}
return result
}
func HTMLText(s string) string {
tokenizer := xhtml.NewTokenizer(strings.NewReader(s))
contents := make([]string, 0)
for {
token := tokenizer.Next()
if token == xhtml.ErrorToken {
break
}
if token == xhtml.TextToken {
content := strings.TrimSpace(xhtml.UnescapeString(string(tokenizer.Text())))
if len(content) > 0 {
contents = append(contents, content)
}
}
}
return strings.Join(contents, " ")
}
func (s *Storage) SyncSearch() {
rows, err := s.db.Query(`
select id, title, content, description
from items
where search_rowid is null;
`)
if err != nil {
s.log.Print(err)
return
}
items := make([]Item, 0)
for rows.Next() {
var item Item
rows.Scan(&item.Id, &item.Title, &item.Content, &item.Description)
items = append(items, item)
}
for _, item := range items {
result, err := s.db.Exec(`
insert into search (title, description, content) values (?, ?, ?)`,
item.Title, HTMLText(item.Description), HTMLText(item.Content),
)
if err != nil {
s.log.Print(err)
return
}
if numrows, err := result.RowsAffected(); err == nil && numrows == 1 {
if rowId, err := result.LastInsertId(); err == nil {
s.db.Exec(
`update items set search_rowid = ? where id = ?`,
rowId, item.Id,
)
}
}
}
}
func (s *Storage) DeleteOldItems() {
rows, err := s.db.Query(fmt.Sprintf(`
select feed_id, count(*) as num_items
from items
where status != %d
group by feed_id
having num_items > 50
`, STARRED))
if err != nil {
s.log.Print(err)
return
}
feedIds := make([]int64, 0)
for rows.Next() {
var id int64
rows.Scan(&id, nil)
feedIds = append(feedIds, id)
}
for _, feedId := range feedIds {
result, err := s.db.Exec(`
delete from items where feed_id = ? and status != ? and date_arrived < ?`,
feedId,
STARRED,
time.Now().Add(-time.Hour*24*90), // 90 days
)
if err != nil {
s.log.Print(err)
return
}
num, err := result.RowsAffected()
if err != nil {
s.log.Print(err)
return
}
if num > 0 {
s.log.Printf("Deleted %d old items (%d)", num, feedId)
}
}
}

91
src/storage/settings.go Normal file
View File

@@ -0,0 +1,91 @@
package storage
import "encoding/json"
func settingsDefaults() map[string]interface{} {
return map[string]interface{}{
"filter": "",
"feed": "",
"feed_list_width": 300,
"item_list_width": 300,
"sort_newest_first": true,
"theme_name": "light",
"theme_font": "",
"theme_size": 1,
"refresh_rate": 0,
}
}
func (s *Storage) GetSettingsValue(key string) interface{} {
row := s.db.QueryRow(`select val from settings where key=?`, key)
if row == nil {
return settingsDefaults()[key]
}
var val []byte
row.Scan(&val)
if len(val) == 0 {
return nil
}
var valDecoded interface{}
if err := json.Unmarshal([]byte(val), &valDecoded); err != nil {
s.log.Print(err)
return nil
}
return valDecoded
}
func (s *Storage) GetSettingsValueInt64(key string) int64 {
val := s.GetSettingsValue(key)
if val != nil {
if fval, ok := val.(float64); ok {
return int64(fval)
}
}
return 0
}
func (s *Storage) GetSettings() map[string]interface{} {
result := settingsDefaults()
rows, err := s.db.Query(`select key, val from settings;`)
if err != nil {
s.log.Print(err)
return result
}
for rows.Next() {
var key string
var val []byte
var valDecoded interface{}
rows.Scan(&key, &val)
if err = json.Unmarshal([]byte(val), &valDecoded); err != nil {
s.log.Print(err)
continue
}
result[key] = valDecoded
}
return result
}
func (s *Storage) UpdateSettings(kv map[string]interface{}) bool {
defaults := settingsDefaults()
for key, val := range kv {
if defaults[key] == nil {
continue
}
valEncoded, err := json.Marshal(val)
if err != nil {
s.log.Print(err)
return false
}
_, err = s.db.Exec(`
insert into settings (key, val) values (?, ?)
on conflict (key) do update set val=?`,
key, valEncoded, valEncoded,
)
if err != nil {
s.log.Print(err)
return false
}
}
return true
}

103
src/storage/storage.go Normal file
View File

@@ -0,0 +1,103 @@
package storage
import (
"database/sql"
_ "github.com/mattn/go-sqlite3"
"log"
"os"
)
var initQuery string = `
create table if not exists folders (
id integer primary key autoincrement,
title text not null,
is_expanded boolean not null default false
);
create unique index if not exists idx_folder_title on folders(title);
create table if not exists feeds (
id integer primary key autoincrement,
folder_id references folders(id),
title text not null,
description text,
link text,
feed_link text not null,
icon blob
);
create index if not exists idx_feed_folder_id on feeds(folder_id);
create unique index if not exists idx_feed_feed_link on feeds(feed_link);
create table if not exists items (
id integer primary key autoincrement,
guid string not null,
feed_id references feeds(id),
title text,
link text,
description text,
content text,
author text,
date datetime,
date_updated datetime,
date_arrived datetime,
status integer,
image text,
search_rowid integer
);
create index if not exists idx_item_feed_id on items(feed_id);
create index if not exists idx_item_status on items(status);
create index if not exists idx_item_search_rowid on items(search_rowid);
create unique index if not exists idx_item_guid on items(feed_id, guid);
create table if not exists settings (
key string primary key,
val blob
);
create virtual table if not exists search using fts4(title, description, content);
create trigger if not exists del_item_search after delete on items begin
delete from search where rowid = old.search_rowid;
end;
create table if not exists http_states (
feed_id references feeds(id) unique,
last_refreshed datetime not null,
-- http header fields --
last_modified string not null,
etag string not null
);
create table if not exists feed_errors (
feed_id references feeds(id) unique,
error string
);
`
type Storage struct {
db *sql.DB
log *log.Logger
}
func New(path string, logger *log.Logger) (*Storage, error) {
if _, err := os.Stat(path); err != nil {
if !os.IsNotExist(err) {
return nil, err
}
}
db, err := sql.Open("sqlite3", path)
if err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
if _, err := db.Exec(initQuery); err != nil {
return nil, err
}
return &Storage{db: db, log: logger}, nil
}