7 Commits

Author SHA1 Message Date
nkanaev
f2556178b3 add storage interface, fix all references 2026-06-10 22:24:04 +01:00
nkanaev
3f10371975 fix references 2026-06-09 16:35:14 +01:00
nkanaev
dee386b586 move structs to model 2026-06-09 16:05:56 +01:00
nkanaev
dc836ed4fd rename Storage struct to SQLiteStorage 2026-06-07 23:17:07 +01:00
nkanaev
76adcf0d62 rename package names 2026-06-07 23:15:37 +01:00
nkanaev
f29ad0c20a create sqlite package 2026-06-07 23:14:16 +01:00
nkanaev
14835660fb interface 2026-06-07 23:01:48 +01:00
23 changed files with 506 additions and 451 deletions

View File

@@ -14,7 +14,7 @@ type Middleware struct {
Password string
BasePath string
Public []string
DB *storage.Storage
DB storage.Storage
}
func (m *Middleware) Handler(c *router.Context) {

View File

@@ -14,6 +14,7 @@ import (
"github.com/nkanaev/yarr/src/server/auth"
"github.com/nkanaev/yarr/src/server/router"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/storage/model"
)
type FeverGroup struct {
@@ -61,7 +62,7 @@ func writeFeverJSON(c *router.Context, data map[string]any, lastRefreshed int64)
c.JSON(http.StatusOK, data)
}
func getLastRefreshedOnTime(feedStates []storage.FeedState) int64 {
func getLastRefreshedOnTime(feedStates []model.FeedState) int64 {
var lastRefreshed int64
for _, state := range feedStates {
if state.LastRefreshed.Unix() > lastRefreshed {
@@ -140,7 +141,7 @@ func joinInts(values []int64) string {
return result.String()
}
func feedGroups(db *storage.Storage) []*FeverFeedsGroup {
func feedGroups(db storage.Storage) []*FeverFeedsGroup {
feeds := db.ListFeeds()
groupFeeds := make(map[int64][]int64)
@@ -176,7 +177,7 @@ func (s *Server) feverGroupsHandler(c *router.Context) {
func (s *Server) feverFeedsHandler(c *router.Context) {
feeds := s.db.ListFeeds()
states, _ := s.db.ListFeedStates()
statesMap := make(map[int64]storage.FeedState)
statesMap := make(map[int64]model.FeedState)
for _, state := range states {
statesMap[state.FeedID] = state
}
@@ -230,7 +231,7 @@ func (s *Server) feverFaviconsHandler(c *router.Context) {
const listLimit = 50
func (s *Server) feverItemsHandler(c *router.Context) {
filter := storage.ItemFilter{}
filter := model.ItemFilter{}
query := c.Req.URL.Query()
switch {
@@ -262,11 +263,11 @@ func (s *Server) feverItemsHandler(c *router.Context) {
time := date.Unix()
isSaved := 0
if item.Status == storage.STARRED {
if item.Status == model.STARRED {
isSaved = 1
}
isRead := 0
if item.Status == storage.READ {
if item.Status == model.READ {
isRead = 1
}
feverItems[i] = FeverItem{
@@ -299,10 +300,10 @@ func (s *Server) feverLinksHandler(c *router.Context) {
}
func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
status := storage.UNREAD
status := model.UNREAD
itemIds := make([]int64, 0)
itemFilter := storage.ItemFilter{
itemFilter := model.ItemFilter{
Status: &status,
}
for {
@@ -322,10 +323,10 @@ func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
}
func (s *Server) feverSavedItemIDsHandler(c *router.Context) {
status := storage.STARRED
status := model.STARRED
itemIds := make([]int64, 0)
itemFilter := storage.ItemFilter{
itemFilter := model.ItemFilter{
Status: &status,
}
for {
@@ -353,16 +354,16 @@ func (s *Server) feverMarkHandler(c *router.Context) {
switch c.Req.Form.Get("mark") {
case "item":
var status storage.ItemStatus
var status model.ItemStatus
switch c.Req.Form.Get("as") {
case "read":
status = storage.READ
status = model.READ
case "unread":
status = storage.UNREAD
status = model.UNREAD
case "saved":
status = storage.STARRED
status = model.STARRED
case "unsaved":
status = storage.READ
status = model.READ
default:
c.Out.WriteHeader(http.StatusBadRequest)
return
@@ -372,7 +373,7 @@ func (s *Server) feverMarkHandler(c *router.Context) {
if c.Req.Form.Get("as") != "read" {
c.Out.WriteHeader(http.StatusBadRequest)
}
markFilter := storage.MarkFilter{FeedID: &id}
markFilter := model.MarkFilter{FeedID: &id}
x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
if x > 0 {
before := time.Unix(x, 0).UTC()
@@ -383,7 +384,7 @@ func (s *Server) feverMarkHandler(c *router.Context) {
if c.Req.Form.Get("as") != "read" {
c.Out.WriteHeader(http.StatusBadRequest)
}
markFilter := storage.MarkFilter{}
markFilter := model.MarkFilter{}
if id > 0 {
markFilter.FolderID = &id
}

View File

@@ -1,9 +1,9 @@
package server
import "github.com/nkanaev/yarr/src/storage"
import "github.com/nkanaev/yarr/src/storage/model"
type ItemUpdateForm struct {
Status *storage.ItemStatus `json:"status,omitempty"`
Status *model.ItemStatus `json:"status,omitempty"`
}
type FolderCreateForm struct {

View File

@@ -20,7 +20,7 @@ import (
"github.com/nkanaev/yarr/src/server/gzip"
"github.com/nkanaev/yarr/src/server/opml"
"github.com/nkanaev/yarr/src/server/router"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/storage/model"
"github.com/nkanaev/yarr/src/worker"
)
@@ -141,7 +141,7 @@ func (s *Server) handleFolder(c *router.Context) {
c.Out.WriteHeader(http.StatusBadRequest)
return
}
s.db.UpdateFolder(id, storage.UpdateFolderParams{
s.db.UpdateFolder(id, model.UpdateFolderParams{
Title: body.Title,
IsExpanded: body.IsExpanded,
})
@@ -248,7 +248,7 @@ func (s *Server) handleFeedList(c *router.Context) {
map[string]any{"status": "multiple", "choice": result.Sources},
)
case result.Feed != nil:
feed := s.db.CreateFeed(storage.CreateFeedParams{
feed := s.db.CreateFeed(model.CreateFeedParams{
Title: result.Feed.Title,
Link: result.Feed.SiteURL,
FeedLink: result.FeedLink,
@@ -288,7 +288,7 @@ func (s *Server) handleFeed(c *router.Context) {
c.Out.WriteHeader(http.StatusBadRequest)
return
}
params := storage.UpdateFeedParams{}
params := model.UpdateFeedParams{}
if title, ok := body["title"]; ok {
if reflect.TypeOf(title).Kind() == reflect.String {
t := title.(string)
@@ -297,10 +297,10 @@ func (s *Server) handleFeed(c *router.Context) {
}
if f_id, ok := body["folder_id"]; ok {
if f_id == nil {
params.FolderID = storage.SetNullable[int64](nil)
params.FolderID = model.SetNullable[int64](nil)
} else if reflect.TypeOf(f_id).Kind() == reflect.Float64 {
folderId := int64(f_id.(float64))
params.FolderID = storage.SetNullable(&folderId)
params.FolderID = model.SetNullable(&folderId)
}
}
if link, ok := body["feed_link"]; ok {
@@ -366,7 +366,7 @@ func (s *Server) handleItemList(c *router.Context) {
perPage := 20
query := c.Req.URL.Query()
filter := storage.ItemFilter{}
filter := model.ItemFilter{}
if folderID, err := c.QueryInt64("folder_id"); err == nil {
filter.FolderID = &folderID
}
@@ -377,7 +377,7 @@ func (s *Server) handleItemList(c *router.Context) {
filter.After = &after
}
if status := query.Get("status"); len(status) != 0 {
statusValue := storage.StatusValues[status]
statusValue := model.StatusValues[status]
filter.Status = &statusValue
}
if search := query.Get("search"); len(search) != 0 {
@@ -403,7 +403,7 @@ func (s *Server) handleItemList(c *router.Context) {
"has_more": hasMore,
})
} else if c.Req.Method == "PUT" {
filter := storage.MarkFilter{}
filter := model.MarkFilter{}
if folderID, err := c.QueryInt64("folder_id"); err == nil {
filter.FolderID = &folderID
@@ -422,7 +422,7 @@ func (s *Server) handleSettings(c *router.Context) {
if c.Req.Method == "GET" {
c.JSON(http.StatusOK, s.db.GetSettings())
} else if c.Req.Method == "PUT" {
var params storage.UpdateSettingsParams
var params model.UpdateSettingsParams
if err := json.NewDecoder(c.Req.Body).Decode(&params); err != nil {
c.Out.WriteHeader(http.StatusBadRequest)
return
@@ -452,7 +452,7 @@ func (s *Server) handleOPMLImport(c *router.Context) {
return
}
for _, f := range doc.Feeds {
s.db.CreateFeed(storage.CreateFeedParams{
s.db.CreateFeed(model.CreateFeedParams{
Title: f.Title,
Link: f.SiteUrl,
FeedLink: f.FeedUrl,
@@ -461,7 +461,7 @@ func (s *Server) handleOPMLImport(c *router.Context) {
for _, f := range doc.Folders {
folder := s.db.CreateFolder(f.Title)
for _, ff := range f.AllFeeds() {
s.db.CreateFeed(storage.CreateFeedParams{
s.db.CreateFeed(model.CreateFeedParams{
Title: ff.Title,
Link: ff.SiteUrl,
FeedLink: ff.FeedUrl,
@@ -485,7 +485,7 @@ func (s *Server) handleOPMLExport(c *router.Context) {
doc := opml.Folder{}
feedsByFolderID := make(map[int64][]*storage.Feed)
feedsByFolderID := make(map[int64][]*model.Feed)
for _, feed := range s.db.ListFeeds() {
if feed.FolderId == nil {
doc.Feeds = append(doc.Feeds, opml.Feed{

View File

@@ -11,6 +11,7 @@ import (
"testing"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/storage/model"
)
func TestStatic(t *testing.T) {
@@ -79,8 +80,8 @@ func TestFeedIcons(t *testing.T) {
log.SetOutput(io.Discard)
db, _ := storage.New(":memory:")
icon := []byte("test")
feed := db.CreateFeed(storage.CreateFeedParams{})
db.UpdateFeed(feed.Id, storage.UpdateFeedParams{Icon: storage.SetNullable(&icon)})
feed := db.CreateFeed(model.CreateFeedParams{})
db.UpdateFeed(feed.Id, model.UpdateFeedParams{Icon: model.SetNullable(&icon)})
log.SetOutput(os.Stderr)
recorder := httptest.NewRecorder()

View File

@@ -14,7 +14,7 @@ import (
type Server struct {
Addr string
db *storage.Storage
db storage.Storage
worker *worker.Worker
cache map[string]any
cache_mutex *sync.Mutex
@@ -29,7 +29,7 @@ type Server struct {
KeyFile string
}
func NewServer(db *storage.Storage, addr string) *Server {
func NewServer(db storage.Storage, addr string) *Server {
return &Server{
db: db,
Addr: addr,

188
src/storage/model/model.go Normal file
View File

@@ -0,0 +1,188 @@
package model
import (
"encoding/json"
"time"
)
type Feed struct {
Id int64 `json:"id"`
FolderId *int64 `json:"folder_id"`
Title string `json:"title"`
Description string `json:"description"`
Link string `json:"link"`
FeedLink string `json:"feed_link"`
Icon *[]byte `json:"icon,omitempty"`
HasIcon bool `json:"has_icon"`
}
type CreateFeedParams struct {
Title string
Description string
Link string
FeedLink string
FolderID *int64
}
type Item struct {
Id int64 `json:"id"`
GUID string `json:"guid"`
FeedId int64 `json:"feed_id"`
Title string `json:"title"`
Link string `json:"link"`
Content string `json:"content,omitempty"`
Date time.Time `json:"date"`
Status ItemStatus `json:"status"`
MediaLinks MediaLinks `json:"media_links"`
}
type ItemStatus int
const (
UNREAD ItemStatus = 0
READ ItemStatus = 1
STARRED ItemStatus = 2
)
var StatusRepresentations = map[ItemStatus]string{
UNREAD: "unread",
READ: "read",
STARRED: "starred",
}
var StatusValues = map[string]ItemStatus{
"unread": UNREAD,
"read": READ,
"starred": STARRED,
}
func (s ItemStatus) MarshalJSON() ([]byte, error) {
return json.Marshal(StatusRepresentations[s])
}
func (s *ItemStatus) UnmarshalJSON(b []byte) error {
var str string
if err := json.Unmarshal(b, &str); err != nil {
return err
}
*s = StatusValues[str]
return nil
}
type MediaLink struct {
URL string `json:"url"`
Type string `json:"type"`
Description string `json:"description,omitempty"`
}
type MediaLinks []MediaLink
type ItemFilter struct {
FolderID *int64
FeedID *int64
Status *ItemStatus
Search *string
After *int64
IDs *[]int64
SinceID *int64
MaxID *int64
Before *time.Time
}
type MarkFilter struct {
FolderID *int64
FeedID *int64
Before *time.Time
}
type Folder struct {
Id int64 `json:"id"`
Title string `json:"title"`
IsExpanded bool `json:"is_expanded"`
}
type UpdateFolderParams struct {
Title *string
IsExpanded *bool
}
type FeedStat struct {
FeedId int64 `json:"feed_id"`
UnreadCount int64 `json:"unread"`
StarredCount int64 `json:"starred"`
}
type Settings struct {
Filter string `json:"filter"`
Feed string `json:"feed"`
FeedListWidth int `json:"feed_list_width"`
ItemListWidth int `json:"item_list_width"`
SortNewestFirst bool `json:"sort_newest_first"`
ThemeName string `json:"theme_name"`
ThemeFont string `json:"theme_font"`
ThemeSize int `json:"theme_size"`
RefreshRate int64 `json:"refresh_rate"`
Language string `json:"language"`
}
type UpdateSettingsParams struct {
Filter *string `json:"filter"`
Feed *string `json:"feed"`
FeedListWidth *int `json:"feed_list_width"`
ItemListWidth *int `json:"item_list_width"`
SortNewestFirst *bool `json:"sort_newest_first"`
ThemeName *string `json:"theme_name"`
ThemeFont *string `json:"theme_font"`
ThemeSize *int `json:"theme_size"`
RefreshRate *int64 `json:"refresh_rate"`
Language *string `json:"language"`
}
func (s Settings) Map() map[string]any {
return map[string]any{
"filter": s.Filter,
"feed": s.Feed,
"feed_list_width": s.FeedListWidth,
"item_list_width": s.ItemListWidth,
"sort_newest_first": s.SortNewestFirst,
"theme_name": s.ThemeName,
"theme_font": s.ThemeFont,
"theme_size": s.ThemeSize,
"refresh_rate": s.RefreshRate,
"language": s.Language,
}
}
type FeedState struct {
FeedID int64
LastRefreshed time.Time
LastError string
HTTPLastModified string
HTTPEtag string
}
type UpdateFeedStateParams struct {
LastRefreshed *time.Time
LastError *string
HTTPLastModified *string
HTTPEtag *string
}
type UpdateFeedParams struct {
Title *string
FeedLink *string
FolderID Nullable[int64]
Icon Nullable[[]byte]
}
type Nullable[T any] struct {
Set bool
Value *T
}
func SetNullable[T any](v *T) Nullable[T] {
return Nullable[T]{Set: true, Value: v}
}

View File

@@ -1,30 +1,13 @@
package storage
package sqlite
import (
"database/sql"
"log"
"github.com/nkanaev/yarr/src/storage/model"
)
type Feed struct {
Id int64 `json:"id"`
FolderId *int64 `json:"folder_id"`
Title string `json:"title"`
Description string `json:"description"`
Link string `json:"link"`
FeedLink string `json:"feed_link"`
Icon *[]byte `json:"icon,omitempty"`
HasIcon bool `json:"has_icon"`
}
type CreateFeedParams struct {
Title string
Description string
Link string
FeedLink string
FolderID *int64
}
func (s *Storage) CreateFeed(params CreateFeedParams) *Feed {
func (s *SQLiteStorage) CreateFeed(params model.CreateFeedParams) *model.Feed {
title := params.Title
if title == "" {
title = params.FeedLink
@@ -47,7 +30,7 @@ func (s *Storage) CreateFeed(params CreateFeedParams) *Feed {
log.Print(err)
return nil
}
return &Feed{
return &model.Feed{
Id: id,
Title: title,
Description: params.Description,
@@ -57,7 +40,7 @@ func (s *Storage) CreateFeed(params CreateFeedParams) *Feed {
}
}
func (s *Storage) DeleteFeed(feedId int64) bool {
func (s *SQLiteStorage) DeleteFeed(feedId int64) bool {
result, err := s.db.Exec(`delete from feeds where id = :id`, sql.Named("id", feedId))
if err != nil {
log.Print(err)
@@ -73,14 +56,7 @@ func (s *Storage) DeleteFeed(feedId int64) bool {
return nrows == 1
}
type UpdateFeedParams struct {
Title *string
FeedLink *string
FolderID Nullable[int64]
Icon Nullable[[]byte]
}
func (s *Storage) UpdateFeed(feedId int64, params UpdateFeedParams) (bool, error) {
func (s *SQLiteStorage) UpdateFeed(feedId int64, params model.UpdateFeedParams) (bool, error) {
_, err := s.db.Exec(`
update feeds set
title = coalesce(:title, title),
@@ -104,8 +80,8 @@ func (s *Storage) UpdateFeed(feedId int64, params UpdateFeedParams) (bool, error
return true, nil
}
func (s *Storage) ListFeeds() []Feed {
result := make([]Feed, 0)
func (s *SQLiteStorage) ListFeeds() []model.Feed {
result := make([]model.Feed, 0)
rows, err := s.db.Query(`
select id, folder_id, title, description, link, feed_link,
ifnull(length(icon), 0) > 0 as has_icon
@@ -117,7 +93,7 @@ func (s *Storage) ListFeeds() []Feed {
return result
}
for rows.Next() {
var f Feed
var f model.Feed
err = rows.Scan(
&f.Id,
&f.FolderId,
@@ -136,8 +112,8 @@ func (s *Storage) ListFeeds() []Feed {
return result
}
func (s *Storage) GetFeed(id int64) *Feed {
var f Feed
func (s *SQLiteStorage) GetFeed(id int64) *model.Feed {
var f model.Feed
err := s.db.QueryRow(`
select
id, folder_id, title, link, feed_link,

View File

@@ -1,13 +1,15 @@
package storage
package sqlite
import (
"reflect"
"testing"
"github.com/nkanaev/yarr/src/storage/model"
)
func TestCreateFeed(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
feed1 := db.CreateFeed(model.CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
if feed1 == nil || feed1.Id == 0 {
t.Fatal("expected feed")
}
@@ -19,16 +21,16 @@ func TestCreateFeed(t *testing.T) {
func TestCreateFeedSameLink(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", FeedLink: "http://example1.com/feed.xml"})
feed1 := db.CreateFeed(model.CreateFeedParams{Title: "title", FeedLink: "http://example1.com/feed.xml"})
if feed1 == nil || feed1.Id == 0 {
t.Fatal("expected feed")
}
for range 10 {
db.CreateFeed(CreateFeedParams{Title: "title", FeedLink: "http://example2.com/feed.xml"})
db.CreateFeed(model.CreateFeedParams{Title: "title", FeedLink: "http://example2.com/feed.xml"})
}
feed2 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example1.com/feed.xml"})
feed2 := db.CreateFeed(model.CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example1.com/feed.xml"})
if feed1.Id != feed2.Id {
t.Fatalf("expected the same feed.\nwant: %#v\nhave: %#v", feed1, feed2)
}
@@ -40,25 +42,25 @@ func TestReadFeed(t *testing.T) {
t.Fatal("cannot get nonexistent feed")
}
feed1 := db.CreateFeed(CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
feed2 := db.CreateFeed(CreateFeedParams{Title: "feed 2", Link: "http://example2.com", FeedLink: "http://example2.com/feed.xml"})
feed1 := db.CreateFeed(model.CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
feed2 := db.CreateFeed(model.CreateFeedParams{Title: "feed 2", Link: "http://example2.com", FeedLink: "http://example2.com/feed.xml"})
feeds := db.ListFeeds()
if !reflect.DeepEqual(feeds, []Feed{*feed1, *feed2}) {
if !reflect.DeepEqual(feeds, []model.Feed{*feed1, *feed2}) {
t.Fatalf("invalid feed list: %#v", feeds)
}
}
func TestUpdateFeed(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
feed1 := db.CreateFeed(model.CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
folder := db.CreateFolder("test")
icon := []byte("icon")
title := "newtitle"
db.UpdateFeed(feed1.Id, UpdateFeedParams{
db.UpdateFeed(feed1.Id, model.UpdateFeedParams{
Title: &title,
FolderID: SetNullable(&folder.Id),
Icon: SetNullable(&icon),
FolderID: model.SetNullable(&folder.Id),
Icon: model.SetNullable(&icon),
})
feed2 := db.GetFeed(feed1.Id)
@@ -75,7 +77,7 @@ func TestUpdateFeed(t *testing.T) {
func TestDeleteFeed(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
feed1 := db.CreateFeed(model.CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
if db.DeleteFeed(100500) {
t.Error("cannot delete what does not exist")

View File

@@ -1,19 +1,12 @@
package storage
package sqlite
import (
"database/sql"
"time"
"github.com/nkanaev/yarr/src/storage/model"
)
type FeedState struct {
FeedID int64
LastRefreshed time.Time
LastError string
HTTPLastModified string
HTTPEtag string
}
func (s *Storage) ListFeedStates() ([]FeedState, error) {
func (s *SQLiteStorage) ListFeedStates() ([]model.FeedState, error) {
rows, err := s.db.Query(`
select
feed_id
@@ -28,9 +21,9 @@ func (s *Storage) ListFeedStates() ([]FeedState, error) {
}
defer rows.Close()
states := make([]FeedState, 0)
states := make([]model.FeedState, 0)
for rows.Next() {
var state FeedState
var state model.FeedState
err := rows.Scan(
&state.FeedID,
&state.LastRefreshed,
@@ -46,8 +39,8 @@ func (s *Storage) ListFeedStates() ([]FeedState, error) {
return states, nil
}
func (s *Storage) GetFeedState(feedID int64) (*FeedState, error) {
var state FeedState
func (s *SQLiteStorage) GetFeedState(feedID int64) (*model.FeedState, error) {
var state model.FeedState
err := s.db.QueryRow(`
select
feed_id
@@ -72,14 +65,7 @@ func (s *Storage) GetFeedState(feedID int64) (*FeedState, error) {
return &state, nil
}
type UpdateFeedStateParams struct {
LastRefreshed *time.Time
LastError *string
HTTPLastModified *string
HTTPEtag *string
}
func (s *Storage) UpdateFeedState(feedID int64, params UpdateFeedStateParams) (bool, error) {
func (s *SQLiteStorage) UpdateFeedState(feedID int64, params model.UpdateFeedStateParams) (bool, error) {
lastError := params.LastError
if lastError != nil && *lastError == "" {
lastError = nil

View File

@@ -1,22 +1,24 @@
package storage
package sqlite
import (
"testing"
"time"
"github.com/nkanaev/yarr/src/storage/model"
)
func TestUpdateFeedState_Full(t *testing.T) {
s := testDB()
defer s.Close()
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
f := s.CreateFeed(model.CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
now := time.Now().UTC().Truncate(time.Second)
errMsg := "error"
lmod := "today"
etag := "v1"
ok, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
ok, err := s.UpdateFeedState(f.Id, model.UpdateFeedStateParams{
LastRefreshed: &now,
LastError: &errMsg,
HTTPLastModified: &lmod,
@@ -54,12 +56,12 @@ func TestUpdateFeedState_Partial(t *testing.T) {
s := testDB()
defer s.Close()
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
f := s.CreateFeed(model.CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
etag := "v1"
s.UpdateFeedState(f.Id, UpdateFeedStateParams{HTTPEtag: &etag})
s.UpdateFeedState(f.Id, model.UpdateFeedStateParams{HTTPEtag: &etag})
newErr := "new error"
_, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
_, err := s.UpdateFeedState(f.Id, model.UpdateFeedStateParams{
LastError: &newErr,
})
if err != nil {
@@ -82,12 +84,12 @@ func TestUpdateFeedState_ClearError(t *testing.T) {
s := testDB()
defer s.Close()
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
f := s.CreateFeed(model.CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
errMsg := "error"
s.UpdateFeedState(f.Id, UpdateFeedStateParams{LastError: &errMsg})
s.UpdateFeedState(f.Id, model.UpdateFeedStateParams{LastError: &errMsg})
empty := ""
_, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
_, err := s.UpdateFeedState(f.Id, model.UpdateFeedStateParams{
LastError: &empty,
})
if err != nil {
@@ -107,12 +109,12 @@ func TestListFeedStates(t *testing.T) {
s := testDB()
defer s.Close()
f1 := s.CreateFeed(CreateFeedParams{Title: "F1", FeedLink: "L1"})
f2 := s.CreateFeed(CreateFeedParams{Title: "F2", FeedLink: "L2"})
f1 := s.CreateFeed(model.CreateFeedParams{Title: "F1", FeedLink: "L1"})
f2 := s.CreateFeed(model.CreateFeedParams{Title: "F2", FeedLink: "L2"})
errMsg := "fail"
s.UpdateFeedState(f1.Id, UpdateFeedStateParams{LastError: &errMsg})
s.UpdateFeedState(f2.Id, UpdateFeedStateParams{HTTPEtag: ptr("e")})
s.UpdateFeedState(f1.Id, model.UpdateFeedStateParams{LastError: &errMsg})
s.UpdateFeedState(f2.Id, model.UpdateFeedStateParams{HTTPEtag: ptr("e")})
states, err := s.ListFeedStates()
if err != nil {

View File

@@ -1,17 +1,13 @@
package storage
package sqlite
import (
"database/sql"
"log"
"github.com/nkanaev/yarr/src/storage/model"
)
type Folder struct {
Id int64 `json:"id"`
Title string `json:"title"`
IsExpanded bool `json:"is_expanded"`
}
func (s *Storage) CreateFolder(title string) *Folder {
func (s *SQLiteStorage) CreateFolder(title string) *model.Folder {
expanded := true
row := s.db.QueryRow(`
insert into folders (title, is_expanded) values (:title, :is_expanded)
@@ -27,10 +23,10 @@ func (s *Storage) CreateFolder(title string) *Folder {
log.Print(err)
return nil
}
return &Folder{Id: id, Title: title, IsExpanded: expanded}
return &model.Folder{Id: id, Title: title, IsExpanded: expanded}
}
func (s *Storage) DeleteFolder(folderId int64) bool {
func (s *SQLiteStorage) DeleteFolder(folderId int64) bool {
_, err := s.db.Exec(`delete from folders where id = :id`, sql.Named("id", folderId))
if err != nil {
log.Print(err)
@@ -38,12 +34,7 @@ func (s *Storage) DeleteFolder(folderId int64) bool {
return err == nil
}
type UpdateFolderParams struct {
Title *string
IsExpanded *bool
}
func (s *Storage) UpdateFolder(folderId int64, params UpdateFolderParams) (bool, error) {
func (s *SQLiteStorage) UpdateFolder(folderId int64, params model.UpdateFolderParams) (bool, error) {
_, err := s.db.Exec(`
update folders set
title = coalesce(:title, title),
@@ -61,8 +52,8 @@ func (s *Storage) UpdateFolder(folderId int64, params UpdateFolderParams) (bool,
return true, nil
}
func (s *Storage) ListFolders() []Folder {
result := make([]Folder, 0)
func (s *SQLiteStorage) ListFolders() []model.Folder {
result := make([]model.Folder, 0)
rows, err := s.db.Query(`
select id, title, is_expanded
from folders
@@ -73,7 +64,7 @@ func (s *Storage) ListFolders() []Folder {
return result
}
for rows.Next() {
var f Folder
var f model.Folder
err = rows.Scan(&f.Id, &f.Title, &f.IsExpanded)
if err != nil {
log.Print(err)

View File

@@ -1,7 +1,9 @@
package storage
package sqlite
import (
"testing"
"github.com/nkanaev/yarr/src/storage/model"
)
func TestUpdateFolder(t *testing.T) {
@@ -13,7 +15,7 @@ func TestUpdateFolder(t *testing.T) {
t.Run("rename only", func(t *testing.T) {
newTitle := "new title"
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
ok, err := db.UpdateFolder(folder.Id, model.UpdateFolderParams{
Title: &newTitle,
})
if !ok || err != nil {
@@ -31,7 +33,7 @@ func TestUpdateFolder(t *testing.T) {
t.Run("toggle expanded only", func(t *testing.T) {
isExpanded := false
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
ok, err := db.UpdateFolder(folder.Id, model.UpdateFolderParams{
IsExpanded: &isExpanded,
})
if !ok || err != nil {
@@ -50,7 +52,7 @@ func TestUpdateFolder(t *testing.T) {
t.Run("update both", func(t *testing.T) {
bothTitle := "both"
isExpanded := true
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
ok, err := db.UpdateFolder(folder.Id, model.UpdateFolderParams{
Title: &bothTitle,
IsExpanded: &isExpanded,
})
@@ -65,7 +67,7 @@ func TestUpdateFolder(t *testing.T) {
})
t.Run("update none", func(t *testing.T) {
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{})
ok, err := db.UpdateFolder(folder.Id, model.UpdateFolderParams{})
if !ok || err != nil {
t.Fatalf("UpdateFolder failed: %v", err)
}

View File

@@ -1,56 +1,20 @@
package storage
package sqlite
import (
"cmp"
"database/sql"
"database/sql/driver"
"encoding/json"
"fmt"
"log"
"sort"
"slices"
"strings"
"time"
"github.com/nkanaev/yarr/src/storage/model"
)
type ItemStatus int
const (
UNREAD ItemStatus = 0
READ ItemStatus = 1
STARRED ItemStatus = 2
)
var StatusRepresentations = map[ItemStatus]string{
UNREAD: "unread",
READ: "read",
STARRED: "starred",
}
var StatusValues = map[string]ItemStatus{
"unread": UNREAD,
"read": READ,
"starred": STARRED,
}
func (s ItemStatus) MarshalJSON() ([]byte, error) {
return json.Marshal(StatusRepresentations[s])
}
func (s *ItemStatus) UnmarshalJSON(b []byte) error {
var str string
if err := json.Unmarshal(b, &str); err != nil {
return err
}
*s = StatusValues[str]
return nil
}
type MediaLink struct {
URL string `json:"url"`
Type string `json:"type"`
Description string `json:"description,omitempty"`
}
type MediaLinks []MediaLink
type MediaLinks model.MediaLinks
func (m *MediaLinks) Scan(src any) error {
switch data := src.(type) {
@@ -67,56 +31,7 @@ func (m MediaLinks) Value() (driver.Value, error) {
return json.Marshal(m)
}
type Item struct {
Id int64 `json:"id"`
GUID string `json:"guid"`
FeedId int64 `json:"feed_id"`
Title string `json:"title"`
Link string `json:"link"`
Content string `json:"content,omitempty"`
Date time.Time `json:"date"`
Status ItemStatus `json:"status"`
MediaLinks MediaLinks `json:"media_links"`
}
type ItemFilter struct {
FolderID *int64
FeedID *int64
Status *ItemStatus
Search *string
After *int64
IDs *[]int64
SinceID *int64
MaxID *int64
Before *time.Time
}
type MarkFilter struct {
FolderID *int64
FeedID *int64
Before *time.Time
}
type ItemList []Item
func (list ItemList) Len() int {
return len(list)
}
func (list ItemList) SortKey(i int) string {
return list[i].Date.Format(time.RFC3339) + "::" + list[i].GUID
}
func (list ItemList) Less(i, j int) bool {
return list.SortKey(i) < list.SortKey(j)
}
func (list ItemList) Swap(i, j int) {
list[i], list[j] = list[j], list[i]
}
func (s *Storage) CreateItems(items []Item) bool {
func (s *SQLiteStorage) CreateItems(items []model.Item) bool {
tx, err := s.db.Begin()
if err != nil {
log.Print(err)
@@ -125,10 +40,13 @@ func (s *Storage) CreateItems(items []Item) bool {
now := time.Now().UTC()
itemsSorted := ItemList(items)
sort.Sort(itemsSorted)
slices.SortStableFunc(items, func(a, b model.Item) int {
sa := a.Date.Format(time.RFC3339) + "::" + a.GUID
sb := b.Date.Format(time.RFC3339) + "::" + b.GUID
return cmp.Compare(sa, sb)
})
for _, item := range itemsSorted {
for _, item := range items {
_, err = tx.Exec(`
insert into items (
guid, feed_id, title, link, date,
@@ -148,10 +66,10 @@ func (s *Storage) CreateItems(items []Item) bool {
sql.Named("link", item.Link),
sql.Named("date", item.Date),
sql.Named("content", item.Content),
sql.Named("media_links", item.MediaLinks),
sql.Named("media_links", MediaLinks(item.MediaLinks)),
sql.Named("date_arrived", now),
sql.Named("last_arrived", now),
sql.Named("status", UNREAD),
sql.Named("status", model.UNREAD),
)
if err != nil {
log.Print(err)
@@ -169,7 +87,7 @@ func (s *Storage) CreateItems(items []Item) bool {
return true
}
func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []any) {
func listQueryPredicate(filter model.ItemFilter, newestFirst bool) (string, []any) {
cond := make([]string, 0)
args := make([]any, 0)
if filter.FolderID != nil {
@@ -241,7 +159,7 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []any) {
return predicate, args
}
func (s *Storage) CountItems() int {
func (s *SQLiteStorage) CountItems() int {
var count int
err := s.db.QueryRow(`select count(*) from items`).Scan(&count)
if err != nil {
@@ -251,14 +169,14 @@ func (s *Storage) CountItems() int {
return count
}
func (s *Storage) ListItems(
filter ItemFilter,
func (s *SQLiteStorage) ListItems(
filter model.ItemFilter,
limit int,
newestFirst bool,
withContent bool,
) []Item {
) []model.Item {
predicate, args := listQueryPredicate(filter, newestFirst)
result := make([]Item, 0)
result := make([]model.Item, 0)
order := "date desc, id desc"
if !newestFirst {
@@ -290,11 +208,11 @@ func (s *Storage) ListItems(
return result
}
for rows.Next() {
var x Item
var x model.Item
err = rows.Scan(
&x.Id, &x.GUID, &x.FeedId,
&x.Title, &x.Link, &x.Date,
&x.Status, &x.MediaLinks, &x.Content,
&x.Status, (*MediaLinks)(&x.MediaLinks), &x.Content,
)
if err != nil {
log.Print(err)
@@ -305,8 +223,8 @@ func (s *Storage) ListItems(
return result
}
func (s *Storage) GetItem(id int64) *Item {
i := &Item{}
func (s *SQLiteStorage) GetItem(id int64) *model.Item {
i := &model.Item{}
err := s.db.QueryRow(`
select
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
@@ -315,7 +233,7 @@ func (s *Storage) GetItem(id int64) *Item {
where i.id = :id
`, sql.Named("id", id)).Scan(
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
&i.Date, &i.Status, &i.MediaLinks,
&i.Date, &i.Status, (*MediaLinks)(&i.MediaLinks),
)
if err != nil {
log.Print(err)
@@ -324,7 +242,7 @@ func (s *Storage) GetItem(id int64) *Item {
return i
}
func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
func (s *SQLiteStorage) UpdateItemStatus(item_id int64, status model.ItemStatus) bool {
_, err := s.db.Exec(`update items set status = :status where id = :id`,
sql.Named("status", status),
sql.Named("id", item_id),
@@ -332,8 +250,8 @@ func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
return err == nil
}
func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
predicate, args := listQueryPredicate(ItemFilter{
func (s *SQLiteStorage) MarkItemsRead(filter model.MarkFilter) bool {
predicate, args := listQueryPredicate(model.ItemFilter{
FolderID: filter.FolderID,
FeedID: filter.FeedID,
Before: filter.Before,
@@ -341,7 +259,7 @@ func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
query := fmt.Sprintf(`
update items as i set status = %d
where %s and i.status != %d
`, READ, predicate, STARRED)
`, model.READ, predicate, model.STARRED)
_, err := s.db.Exec(query, args...)
if err != nil {
log.Print(err)
@@ -349,14 +267,8 @@ func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
return err == nil
}
type FeedStat struct {
FeedId int64 `json:"feed_id"`
UnreadCount int64 `json:"unread"`
StarredCount int64 `json:"starred"`
}
func (s *Storage) FeedStats() []FeedStat {
result := make([]FeedStat, 0)
func (s *SQLiteStorage) FeedStats() []model.FeedStat {
result := make([]model.FeedStat, 0)
rows, err := s.db.Query(fmt.Sprintf(`
select
feed_id,
@@ -364,13 +276,13 @@ func (s *Storage) FeedStats() []FeedStat {
sum(case status when %d then 1 else 0 end)
from items
group by feed_id
`, UNREAD, STARRED))
`, model.UNREAD, model.STARRED))
if err != nil {
log.Print(err)
return result
}
for rows.Next() {
stat := FeedStat{}
stat := model.FeedStat{}
rows.Scan(&stat.FeedId, &stat.UnreadCount, &stat.StarredCount)
result = append(result, stat)
}
@@ -388,7 +300,7 @@ var (
// - Never delete starred entries.
// - Keep at least 50 latest items for each feed.
// - Delete entries older than 90 days relative to the latest arrived item in the same feed.
func (s *Storage) DeleteOldItems() {
func (s *SQLiteStorage) DeleteOldItems() {
result, err := s.db.Exec(`
delete from items
where id in (
@@ -405,7 +317,7 @@ func (s *Storage) DeleteOldItems() {
where rn > :keep_size
and last_arrived < datetime(max_la, :keep_days_limit)
)`,
sql.Named("starred_status", STARRED),
sql.Named("starred_status", model.STARRED),
sql.Named("keep_size", itemsKeepSize),
sql.Named("keep_days_limit", fmt.Sprintf("-%d days", itemsKeepDays)),
)

View File

@@ -1,4 +1,4 @@
package storage
package sqlite
import (
"database/sql"
@@ -8,6 +8,8 @@ import (
"testing"
"testing/synctest"
"time"
"github.com/nkanaev/yarr/src/storage/model"
)
/*
@@ -30,22 +32,22 @@ import (
*/
type testItemScope struct {
feed11, feed12 *Feed
feed21, feed01 *Feed
folder1, folder2 *Folder
feed11, feed12 *model.Feed
feed21, feed01 *model.Feed
folder1, folder2 *model.Folder
}
func testItemsSetup(db *Storage) testItemScope {
func testItemsSetup(db *SQLiteStorage) testItemScope {
folder1 := db.CreateFolder("folder1")
folder2 := db.CreateFolder("folder2")
feed11 := db.CreateFeed(CreateFeedParams{Title: "feed11", FeedLink: "http://test.com/feed11.xml", FolderID: &folder1.Id})
feed12 := db.CreateFeed(CreateFeedParams{Title: "feed12", FeedLink: "http://test.com/feed12.xml", FolderID: &folder1.Id})
feed21 := db.CreateFeed(CreateFeedParams{Title: "feed21", FeedLink: "http://test.com/feed21.xml", FolderID: &folder2.Id})
feed01 := db.CreateFeed(CreateFeedParams{Title: "feed01", FeedLink: "http://test.com/feed01.xml"})
feed11 := db.CreateFeed(model.CreateFeedParams{Title: "feed11", FeedLink: "http://test.com/feed11.xml", FolderID: &folder1.Id})
feed12 := db.CreateFeed(model.CreateFeedParams{Title: "feed12", FeedLink: "http://test.com/feed12.xml", FolderID: &folder1.Id})
feed21 := db.CreateFeed(model.CreateFeedParams{Title: "feed21", FeedLink: "http://test.com/feed21.xml", FolderID: &folder2.Id})
feed01 := db.CreateFeed(model.CreateFeedParams{Title: "feed01", FeedLink: "http://test.com/feed01.xml"})
now := time.Now()
db.CreateItems([]Item{
db.CreateItems([]model.Item{
// feed11
{GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)},
{
@@ -98,11 +100,11 @@ func testItemsSetup(db *Storage) testItemScope {
})
db.db.Exec(
`update items set status = :status where guid in ("item112", "item122", "item211", "item012")`,
sql.Named("status", READ),
sql.Named("status", model.READ),
)
db.db.Exec(
`update items set status = :status where guid in ("item113", "item212", "item013")`,
sql.Named("status", STARRED),
sql.Named("status", model.STARRED),
)
return testItemScope{
@@ -115,8 +117,8 @@ func testItemsSetup(db *Storage) testItemScope {
}
}
func getItem(db *Storage, guid string) *Item {
i := &Item{}
func getItem(db *SQLiteStorage, guid string) *model.Item {
i := &model.Item{}
err := db.db.QueryRow(`
select
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
@@ -125,7 +127,7 @@ func getItem(db *Storage, guid string) *Item {
where i.guid = :guid
`, sql.Named("guid", guid)).Scan(
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
&i.Date, &i.Status, &i.MediaLinks,
&i.Date, &i.Status, (*MediaLinks)(&i.MediaLinks),
)
if err != nil {
log.Fatal(err)
@@ -133,7 +135,7 @@ func getItem(db *Storage, guid string) *Item {
return i
}
func getItemGuids(items []Item) []string {
func getItemGuids(items []model.Item) []string {
guids := make([]string, 0)
for _, item := range items {
guids = append(guids, item.GUID)
@@ -147,7 +149,7 @@ func TestListItems(t *testing.T) {
// filter by folder_id
have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
have := getItemGuids(db.ListItems(model.ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
want := []string{"item111", "item112", "item113", "item121", "item122"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -155,7 +157,7 @@ func TestListItems(t *testing.T) {
t.Fail()
}
have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
have = getItemGuids(db.ListItems(model.ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
want = []string{"item211", "item212"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -165,7 +167,7 @@ func TestListItems(t *testing.T) {
// filter by feed_id
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
have = getItemGuids(db.ListItems(model.ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
want = []string{"item111", "item112", "item113"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -173,7 +175,7 @@ func TestListItems(t *testing.T) {
t.Fail()
}
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
have = getItemGuids(db.ListItems(model.ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
want = []string{"item011", "item012", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -183,8 +185,8 @@ func TestListItems(t *testing.T) {
// filter by status
var starred ItemStatus = STARRED
have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
var starred model.ItemStatus = model.STARRED
have = getItemGuids(db.ListItems(model.ItemFilter{Status: &starred}, 10, false, false))
want = []string{"item113", "item212", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -192,8 +194,8 @@ func TestListItems(t *testing.T) {
t.Fail()
}
var unread ItemStatus = UNREAD
have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
var unread model.ItemStatus = model.UNREAD
have = getItemGuids(db.ListItems(model.ItemFilter{Status: &unread}, 10, false, false))
want = []string{"item111", "item121", "item011"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -203,7 +205,7 @@ func TestListItems(t *testing.T) {
// limit
have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
have = getItemGuids(db.ListItems(model.ItemFilter{}, 2, false, false))
want = []string{"item111", "item112"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -213,7 +215,7 @@ func TestListItems(t *testing.T) {
// filter by search
search1 := "title111"
have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &search1}, 4, true, false))
want = []string{"item111"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -222,7 +224,7 @@ func TestListItems(t *testing.T) {
}
// sort by date
have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{}, 4, true, false))
want = []string{"item013", "item012", "item011", "item212"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -239,7 +241,7 @@ func TestListItemsPaginated(t *testing.T) {
item121 := getItem(db, "item121")
// all, newest first
have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
have := getItemGuids(db.ListItems(model.ItemFilter{After: &item012.Id}, 3, true, false))
want := []string{"item011", "item212", "item211"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@@ -248,9 +250,9 @@ func TestListItemsPaginated(t *testing.T) {
}
// unread, newest first
unread := UNREAD
unread := model.UNREAD
have = getItemGuids(
db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false),
db.ListItems(model.ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false),
)
want = []string{"item011", "item121", "item111"}
if !reflect.DeepEqual(have, want) {
@@ -260,9 +262,9 @@ func TestListItemsPaginated(t *testing.T) {
}
// starred, oldest first
starred := STARRED
starred := model.STARRED
have = getItemGuids(
db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false),
db.ListItems(model.ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false),
)
want = []string{"item212", "item013"}
if !reflect.DeepEqual(have, want) {
@@ -274,12 +276,12 @@ func TestListItemsPaginated(t *testing.T) {
func TestMarkItemsRead(t *testing.T) {
// NOTE: starred items must not be marked as read
var read ItemStatus = READ
var read model.ItemStatus = model.READ
db1 := testDB()
testItemsSetup(db1)
db1.MarkItemsRead(MarkFilter{})
have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
db1.MarkItemsRead(model.MarkFilter{})
have := getItemGuids(db1.ListItems(model.ItemFilter{Status: &read}, 10, false, false))
want := []string{
"item111", "item112", "item121", "item122",
"item211", "item011", "item012",
@@ -292,8 +294,8 @@ func TestMarkItemsRead(t *testing.T) {
db2 := testDB()
scope2 := testItemsSetup(db2)
db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
db2.MarkItemsRead(model.MarkFilter{FolderID: &scope2.folder1.Id})
have = getItemGuids(db2.ListItems(model.ItemFilter{Status: &read}, 10, false, false))
want = []string{
"item111", "item112", "item121", "item122",
"item211", "item012",
@@ -306,8 +308,8 @@ func TestMarkItemsRead(t *testing.T) {
db3 := testDB()
scope3 := testItemsSetup(db3)
db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
db3.MarkItemsRead(model.MarkFilter{FeedID: &scope3.feed11.Id})
have = getItemGuids(db3.ListItems(model.ItemFilter{Status: &read}, 10, false, false))
want = []string{
"item111", "item112", "item122",
"item211", "item012",
@@ -321,14 +323,14 @@ func TestMarkItemsRead(t *testing.T) {
func TestDeleteOldItems(t *testing.T) {
now := time.Now().UTC()
starred := STARRED
starred := model.STARRED
t.Run("keeps at least 50 items", func(t *testing.T) {
db := testDB()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]Item, 100)
feed := db.CreateFeed(model.CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]model.Item, 100)
for i := range 100 {
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Hour * 24)}
items[i] = model.Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Hour * 24)}
}
db.CreateItems(items)
@@ -346,10 +348,10 @@ func TestDeleteOldItems(t *testing.T) {
t.Run("keeps all less than 90 days old", func(t *testing.T) {
db := testDB()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]Item, 100)
feed := db.CreateFeed(model.CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]model.Item, 100)
for i := 0; i < 100; i++ {
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
items[i] = model.Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
}
db.CreateItems(items)
@@ -368,10 +370,10 @@ func TestDeleteOldItems(t *testing.T) {
t.Run("keeps starred", func(t *testing.T) {
db := testDB()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]Item, 100)
feed := db.CreateFeed(model.CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]model.Item, 100)
for i := 0; i < 100; i++ {
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
items[i] = model.Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
}
db.CreateItems(items)
@@ -397,9 +399,9 @@ func TestCreateItemsLastArrived(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
db := testDB()
defer db.db.Close()
feed := db.CreateFeed(CreateFeedParams{Title: "test feed", FeedLink: "http://example.com/feed"})
feed := db.CreateFeed(model.CreateFeedParams{Title: "test feed", FeedLink: "http://example.com/feed"})
item := Item{
item := model.Item{
GUID: "item1",
FeedId: feed.Id,
Title: "Title 1",
@@ -407,7 +409,7 @@ func TestCreateItemsLastArrived(t *testing.T) {
}
// 1. Initial creation
db.CreateItems([]Item{item})
db.CreateItems([]model.Item{item})
var lastArrived1 time.Time
err := db.db.QueryRow("select last_arrived from items where guid = ?", item.GUID).Scan(&lastArrived1)
@@ -418,7 +420,7 @@ func TestCreateItemsLastArrived(t *testing.T) {
time.Sleep(time.Second * 10)
// 2. Update on conflict
db.CreateItems([]Item{item})
db.CreateItems([]model.Item{item})
var lastArrived2 time.Time
err = db.db.QueryRow("select last_arrived from items where guid = ?", item.GUID).Scan(&lastArrived2)
@@ -435,9 +437,9 @@ func TestCreateItemsLastArrived(t *testing.T) {
func TestSearch(t *testing.T) {
db := testDB()
defer db.Close()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
feed := db.CreateFeed(model.CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
db.CreateItems([]Item{
db.CreateItems([]model.Item{
{
GUID: "i1",
FeedId: feed.Id,
@@ -460,40 +462,40 @@ func TestSearch(t *testing.T) {
// 1. Basic search
s1 := "emergency"
have := getItemGuids(db.ListItems(ItemFilter{Search: &s1}, 10, true, false))
have := getItemGuids(db.ListItems(model.ItemFilter{Search: &s1}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("basic search failed: expected [i1], got %v", have)
}
// 2. HTML stripping: Should find text, but NOT the tags
s2 := "test"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s2}, 10, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &s2}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("html text search failed: expected [i1], got %v", have)
}
s3 := "secret-class"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s3}, 10, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &s3}, 10, true, false))
if len(have) > 0 {
t.Errorf("html tag search should have failed but found: %v", have)
}
// 3. Multi-word (AND)
s4 := "broadcast system"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s4}, 10, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &s4}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("multi-word search failed: expected [i1], got %v", have)
}
// 4. Unicode
s5 := "Привет"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s5}, 10, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &s5}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i2"}) {
t.Errorf("unicode search failed: expected [i2], got %v", have)
}
s6 := "世界"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s6}, 10, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &s6}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i2"}) {
t.Errorf("unicode search (CJK) failed: expected [i2], got %v", have)
}
@@ -501,14 +503,14 @@ func TestSearch(t *testing.T) {
// 5. Trigger: Update
db.db.Exec("update items set title = 'Updated Title' where guid = 'i1'")
s7 := "Updated"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s7}, 10, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &s7}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("update trigger failed: expected [i1], got %v", have)
}
// 6. Trigger: Delete
db.db.Exec("delete from items where guid = 'i1'")
have = getItemGuids(db.ListItems(ItemFilter{Search: &s7}, 10, true, false))
have = getItemGuids(db.ListItems(model.ItemFilter{Search: &s7}, 10, true, false))
if len(have) > 0 {
t.Errorf("delete trigger failed: found deleted item: %v", have)
}

View File

@@ -1,4 +1,4 @@
package storage
package sqlite
import (
"database/sql"

View File

@@ -1,41 +1,15 @@
package storage
package sqlite
import (
"database/sql"
"encoding/json"
"log"
"github.com/nkanaev/yarr/src/storage/model"
)
type Settings struct {
Filter string `json:"filter"`
Feed string `json:"feed"`
FeedListWidth int `json:"feed_list_width"`
ItemListWidth int `json:"item_list_width"`
SortNewestFirst bool `json:"sort_newest_first"`
ThemeName string `json:"theme_name"`
ThemeFont string `json:"theme_font"`
ThemeSize int `json:"theme_size"`
RefreshRate int64 `json:"refresh_rate"`
Language string `json:"language"`
}
func (s Settings) Map() map[string]any {
return map[string]any{
"filter": s.Filter,
"feed": s.Feed,
"feed_list_width": s.FeedListWidth,
"item_list_width": s.ItemListWidth,
"sort_newest_first": s.SortNewestFirst,
"theme_name": s.ThemeName,
"theme_font": s.ThemeFont,
"theme_size": s.ThemeSize,
"refresh_rate": s.RefreshRate,
"language": s.Language,
}
}
func settingsDefaults() Settings {
return Settings{
func settingsDefaults() model.Settings {
return model.Settings{
Filter: "",
Feed: "",
FeedListWidth: 300,
@@ -49,7 +23,7 @@ func settingsDefaults() Settings {
}
}
func (s *Storage) GetSettings() Settings {
func (s *SQLiteStorage) GetSettings() model.Settings {
result := settingsDefaults()
rows, err := s.db.Query(`select key, val from settings;`)
if err != nil {
@@ -89,20 +63,7 @@ func (s *Storage) GetSettings() Settings {
return result
}
type UpdateSettingsParams struct {
Filter *string `json:"filter"`
Feed *string `json:"feed"`
FeedListWidth *int `json:"feed_list_width"`
ItemListWidth *int `json:"item_list_width"`
SortNewestFirst *bool `json:"sort_newest_first"`
ThemeName *string `json:"theme_name"`
ThemeFont *string `json:"theme_font"`
ThemeSize *int `json:"theme_size"`
RefreshRate *int64 `json:"refresh_rate"`
Language *string `json:"language"`
}
func (s *Storage) UpdateSettings(params UpdateSettingsParams) bool {
func (s *SQLiteStorage) UpdateSettings(params model.UpdateSettingsParams) bool {
tx, err := s.db.Begin()
if err != nil {
log.Print(err)

View File

@@ -1,9 +1,11 @@
package storage
package sqlite
import (
"reflect"
"strings"
"testing"
"github.com/nkanaev/yarr/src/storage/model"
)
func TestSettingsDefaults(t *testing.T) {
@@ -22,7 +24,7 @@ func TestUpdateSettings(t *testing.T) {
s := testDB()
defer s.Close()
params := UpdateSettingsParams{
params := model.UpdateSettingsParams{
ThemeName: ptr("night"),
FeedListWidth: ptr(400),
RefreshRate: ptr(int64(15)),
@@ -49,7 +51,7 @@ func TestGetSettings(t *testing.T) {
s := testDB()
defer s.Close()
s.UpdateSettings(UpdateSettingsParams{Language: ptr("fr")})
s.UpdateSettings(model.UpdateSettingsParams{Language: ptr("fr")})
settings := s.GetSettings()
if settings.Language != "fr" {
@@ -64,8 +66,8 @@ func TestSettingsExhaustive(t *testing.T) {
s := testDB()
defer s.Close()
settingsType := reflect.TypeOf(Settings{})
paramsType := reflect.TypeOf(UpdateSettingsParams{})
settingsType := reflect.TypeOf(model.Settings{})
paramsType := reflect.TypeOf(model.UpdateSettingsParams{})
settings := s.GetSettings()
m := settings.Map()
@@ -125,7 +127,7 @@ func TestSettingsExhaustive(t *testing.T) {
}
}
if ok := s.UpdateSettings(paramsValue.Interface().(UpdateSettingsParams)); !ok {
if ok := s.UpdateSettings(paramsValue.Interface().(model.UpdateSettingsParams)); !ok {
t.Errorf("UpdateSettings failed for %q", jsonKey)
}

View File

@@ -0,0 +1,44 @@
package sqlite
import (
"database/sql"
"log"
"strings"
"github.com/mattn/go-sqlite3"
"github.com/nkanaev/yarr/src/content/htmlutil"
)
func init() {
sql.Register("sqlite3_yarr", &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
return conn.RegisterFunc("strip_html", htmlutil.ExtractText, true)
},
})
}
type SQLiteStorage struct {
db *sql.DB
}
func New(path string) (*SQLiteStorage, error) {
if pos := strings.IndexRune(path, '?'); pos == -1 {
params := "_journal=WAL&_sync=NORMAL&_busy_timeout=5000&cache=shared"
log.Printf("opening db with params: %s", params)
path = path + "?" + params
}
db, err := sql.Open("sqlite3_yarr", path)
if err != nil {
return nil, err
}
if err = migrate(db); err != nil {
return nil, err
}
return &SQLiteStorage{db: db}, nil
}
func (s *SQLiteStorage) Close() error {
return s.db.Close()
}

View File

@@ -1,4 +1,4 @@
package storage
package sqlite
import (
"io"
@@ -7,7 +7,7 @@ import (
"testing"
)
func testDB() *Storage {
func testDB() *SQLiteStorage {
log.SetOutput(io.Discard)
db, err := New(":memory:")
if err != nil {

View File

@@ -1,53 +1,36 @@
package storage
import (
"database/sql"
"log"
"strings"
"github.com/mattn/go-sqlite3"
"github.com/nkanaev/yarr/src/content/htmlutil"
"github.com/nkanaev/yarr/src/storage/model"
"github.com/nkanaev/yarr/src/storage/sqlite"
)
func init() {
sql.Register("sqlite3_yarr", &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
return conn.RegisterFunc("strip_html", htmlutil.ExtractText, true)
},
})
type Storage interface {
Close() error
CountItems() int
CreateFeed(params model.CreateFeedParams) *model.Feed
CreateFolder(title string) *model.Folder
CreateItems(items []model.Item) bool
DeleteFeed(feedId int64) bool
DeleteFolder(folderId int64) bool
DeleteOldItems()
FeedStats() []model.FeedStat
GetFeed(id int64) *model.Feed
GetFeedState(feedID int64) (*model.FeedState, error)
GetItem(id int64) *model.Item
GetSettings() model.Settings
ListFeedStates() ([]model.FeedState, error)
ListFeeds() []model.Feed
ListFolders() []model.Folder
ListItems(filter model.ItemFilter, limit int, newestFirst bool, withContent bool) []model.Item
MarkItemsRead(filter model.MarkFilter) bool
UpdateFeed(feedId int64, params model.UpdateFeedParams) (bool, error)
UpdateFeedState(feedID int64, params model.UpdateFeedStateParams) (bool, error)
UpdateFolder(folderId int64, params model.UpdateFolderParams) (bool, error)
UpdateItemStatus(item_id int64, status model.ItemStatus) bool
UpdateSettings(params model.UpdateSettingsParams) bool
}
type Storage struct {
db *sql.DB
}
type Nullable[T any] struct {
Set bool
Value *T
}
func SetNullable[T any](v *T) Nullable[T] {
return Nullable[T]{Set: true, Value: v}
}
func New(path string) (*Storage, error) {
if pos := strings.IndexRune(path, '?'); pos == -1 {
params := "_journal=WAL&_sync=NORMAL&_busy_timeout=5000&cache=shared"
log.Printf("opening db with params: %s", params)
path = path + "?" + params
}
db, err := sql.Open("sqlite3_yarr", path)
if err != nil {
return nil, err
}
if err = migrate(db); err != nil {
return nil, err
}
return &Storage{db: db}, nil
}
func (s *Storage) Close() error {
return s.db.Close()
func New(path string) (Storage, error) {
return sqlite.New(path)
}

View File

@@ -14,6 +14,7 @@ import (
"github.com/nkanaev/yarr/src/content/scraper"
"github.com/nkanaev/yarr/src/parser"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/storage/model"
"golang.org/x/net/html/charset"
)
@@ -139,28 +140,28 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
return &emptyIcon, nil
}
func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
result := make([]storage.Item, len(items))
func ConvertItems(items []parser.Item, feed model.Feed) []model.Item {
result := make([]model.Item, len(items))
for i, item := range items {
mediaLinks := make(storage.MediaLinks, 0)
mediaLinks := make(model.MediaLinks, 0)
for _, link := range item.MediaLinks {
mediaLinks = append(mediaLinks, storage.MediaLink(link))
mediaLinks = append(mediaLinks, model.MediaLink(link))
}
result[i] = storage.Item{
result[i] = model.Item{
GUID: item.GUID,
FeedId: feed.Id,
Title: item.Title,
Link: item.URL,
Content: item.Content,
Date: item.Date,
Status: storage.UNREAD,
Status: model.UNREAD,
MediaLinks: mediaLinks,
}
}
return result
}
func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
func listItems(f model.Feed, db storage.Storage) ([]model.Item, error) {
lmod := ""
etag := ""
if state, _ := db.GetFeedState(f.Id); state != nil {
@@ -193,7 +194,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
etag = res.Header.Get("Etag")
now := time.Now().UTC()
if lmod != "" || etag != "" {
db.UpdateFeedState(f.Id, storage.UpdateFeedStateParams{
db.UpdateFeedState(f.Id, model.UpdateFeedStateParams{
HTTPLastModified: &lmod,
HTTPEtag: &etag,
LastRefreshed: &now,

View File

@@ -7,19 +7,20 @@ import (
"time"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/storage/model"
)
const NUM_WORKERS = 4
type Worker struct {
db *storage.Storage
db storage.Storage
pending *int32
refresh *time.Ticker
reflock sync.Mutex
stopper chan bool
}
func NewWorker(db *storage.Storage) *Worker {
func NewWorker(db storage.Storage) *Worker {
pending := int32(0)
return &Worker{db: db, pending: &pending}
}
@@ -39,13 +40,13 @@ func (w *Worker) StartFeedCleaner() {
}()
}
func (w *Worker) FindFeedFavicon(feed storage.Feed) {
func (w *Worker) FindFeedFavicon(feed model.Feed) {
icon, err := findFavicon(feed.Link, feed.FeedLink)
if err != nil {
log.Printf("Failed to find favicon for %s (%s): %s", feed.FeedLink, feed.Link, err)
}
if icon != nil {
w.db.UpdateFeed(feed.Id, storage.UpdateFeedParams{Icon: storage.SetNullable(icon)})
w.db.UpdateFeed(feed.Id, model.UpdateFeedParams{Icon: model.SetNullable(icon)})
}
}
@@ -99,11 +100,11 @@ func (w *Worker) RefreshFeeds() {
go w.refresher(feeds)
}
func (w *Worker) refresher(feeds []storage.Feed) {
func (w *Worker) refresher(feeds []model.Feed) {
// w.db.ResetFeedErrors()
srcqueue := make(chan storage.Feed, len(feeds))
dstqueue := make(chan []storage.Item)
srcqueue := make(chan model.Feed, len(feeds))
dstqueue := make(chan []model.Item)
for range NUM_WORKERS {
go w.worker(srcqueue, dstqueue)
@@ -125,15 +126,15 @@ func (w *Worker) refresher(feeds []storage.Feed) {
log.Printf("Finished refreshing %d feeds", len(feeds))
}
func (w *Worker) worker(srcqueue <-chan storage.Feed, dstqueue chan<- []storage.Item) {
func (w *Worker) worker(srcqueue <-chan model.Feed, dstqueue chan<- []model.Item) {
for feed := range srcqueue {
empty := ""
w.db.UpdateFeedState(feed.Id, storage.UpdateFeedStateParams{LastError: &empty})
w.db.UpdateFeedState(feed.Id, model.UpdateFeedStateParams{LastError: &empty})
items, err := listItems(feed, w.db)
if err != nil {
errMsg := err.Error()
w.db.UpdateFeedState(feed.Id, storage.UpdateFeedStateParams{LastError: &errMsg})
w.db.UpdateFeedState(feed.Id, model.UpdateFeedStateParams{LastError: &errMsg})
}
if len(items) > 0 && !feed.HasIcon {
w.FindFeedFavicon(feed)