mirror of
https://github.com/nkanaev/yarr.git
synced 2026-06-24 17:15:17 +00:00
create sqlite package
This commit is contained in:
157
src/storage/sqlite/feed.go
Normal file
157
src/storage/sqlite/feed.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
)
|
||||
|
||||
type Feed struct {
|
||||
Id int64 `json:"id"`
|
||||
FolderId *int64 `json:"folder_id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Link string `json:"link"`
|
||||
FeedLink string `json:"feed_link"`
|
||||
Icon *[]byte `json:"icon,omitempty"`
|
||||
HasIcon bool `json:"has_icon"`
|
||||
}
|
||||
|
||||
type CreateFeedParams struct {
|
||||
Title string
|
||||
Description string
|
||||
Link string
|
||||
FeedLink string
|
||||
FolderID *int64
|
||||
}
|
||||
|
||||
func (s *Storage) CreateFeed(params CreateFeedParams) *Feed {
|
||||
title := params.Title
|
||||
if title == "" {
|
||||
title = params.FeedLink
|
||||
}
|
||||
row := s.db.QueryRow(`
|
||||
insert into feeds (title, description, link, feed_link, folder_id)
|
||||
values (:title, :description, :link, :feed_link, :folder_id)
|
||||
on conflict (feed_link) do update set folder_id = :folder_id
|
||||
returning id`,
|
||||
sql.Named("title", title),
|
||||
sql.Named("description", params.Description),
|
||||
sql.Named("link", params.Link),
|
||||
sql.Named("feed_link", params.FeedLink),
|
||||
sql.Named("folder_id", params.FolderID),
|
||||
)
|
||||
|
||||
var id int64
|
||||
err := row.Scan(&id)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return nil
|
||||
}
|
||||
return &Feed{
|
||||
Id: id,
|
||||
Title: title,
|
||||
Description: params.Description,
|
||||
Link: params.Link,
|
||||
FeedLink: params.FeedLink,
|
||||
FolderId: params.FolderID,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Storage) DeleteFeed(feedId int64) bool {
|
||||
result, err := s.db.Exec(`delete from feeds where id = :id`, sql.Named("id", feedId))
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return false
|
||||
}
|
||||
nrows, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
if err != sql.ErrNoRows {
|
||||
log.Print(err)
|
||||
}
|
||||
return false
|
||||
}
|
||||
return nrows == 1
|
||||
}
|
||||
|
||||
type UpdateFeedParams struct {
|
||||
Title *string
|
||||
FeedLink *string
|
||||
FolderID Nullable[int64]
|
||||
Icon Nullable[[]byte]
|
||||
}
|
||||
|
||||
func (s *Storage) UpdateFeed(feedId int64, params UpdateFeedParams) (bool, error) {
|
||||
_, err := s.db.Exec(`
|
||||
update feeds set
|
||||
title = coalesce(:title, title),
|
||||
feed_link = coalesce(:feed_link, feed_link),
|
||||
folder_id = case when :update_folder_id then :folder_id else folder_id end,
|
||||
icon = case when :update_icon then :icon else icon end
|
||||
where id = :id
|
||||
`,
|
||||
sql.Named("id", feedId),
|
||||
sql.Named("title", params.Title),
|
||||
sql.Named("feed_link", params.FeedLink),
|
||||
sql.Named("update_folder_id", params.FolderID.Set),
|
||||
sql.Named("folder_id", params.FolderID.Value),
|
||||
sql.Named("update_icon", params.Icon.Set),
|
||||
sql.Named("icon", params.Icon.Value),
|
||||
)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Storage) ListFeeds() []Feed {
|
||||
result := make([]Feed, 0)
|
||||
rows, err := s.db.Query(`
|
||||
select id, folder_id, title, description, link, feed_link,
|
||||
ifnull(length(icon), 0) > 0 as has_icon
|
||||
from feeds
|
||||
order by title collate nocase
|
||||
`)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
for rows.Next() {
|
||||
var f Feed
|
||||
err = rows.Scan(
|
||||
&f.Id,
|
||||
&f.FolderId,
|
||||
&f.Title,
|
||||
&f.Description,
|
||||
&f.Link,
|
||||
&f.FeedLink,
|
||||
&f.HasIcon,
|
||||
)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
result = append(result, f)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *Storage) GetFeed(id int64) *Feed {
|
||||
var f Feed
|
||||
err := s.db.QueryRow(`
|
||||
select
|
||||
id, folder_id, title, link, feed_link,
|
||||
icon, ifnull(icon, '') != '' as has_icon
|
||||
from feeds where id = :id
|
||||
`, sql.Named("id", id)).Scan(
|
||||
&f.Id, &f.FolderId, &f.Title, &f.Link, &f.FeedLink,
|
||||
&f.Icon, &f.HasIcon,
|
||||
)
|
||||
if err != nil {
|
||||
if err != sql.ErrNoRows {
|
||||
log.Print(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return &f
|
||||
}
|
||||
90
src/storage/sqlite/feed_test.go
Normal file
90
src/storage/sqlite/feed_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCreateFeed(t *testing.T) {
|
||||
db := testDB()
|
||||
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
|
||||
if feed1 == nil || feed1.Id == 0 {
|
||||
t.Fatal("expected feed")
|
||||
}
|
||||
feed2 := db.GetFeed(feed1.Id)
|
||||
if feed2 == nil || !reflect.DeepEqual(feed1, feed2) {
|
||||
t.Fatal("invalid feed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateFeedSameLink(t *testing.T) {
|
||||
db := testDB()
|
||||
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", FeedLink: "http://example1.com/feed.xml"})
|
||||
if feed1 == nil || feed1.Id == 0 {
|
||||
t.Fatal("expected feed")
|
||||
}
|
||||
|
||||
for range 10 {
|
||||
db.CreateFeed(CreateFeedParams{Title: "title", FeedLink: "http://example2.com/feed.xml"})
|
||||
}
|
||||
|
||||
feed2 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example1.com/feed.xml"})
|
||||
if feed1.Id != feed2.Id {
|
||||
t.Fatalf("expected the same feed.\nwant: %#v\nhave: %#v", feed1, feed2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadFeed(t *testing.T) {
|
||||
db := testDB()
|
||||
if db.GetFeed(100500) != nil {
|
||||
t.Fatal("cannot get nonexistent feed")
|
||||
}
|
||||
|
||||
feed1 := db.CreateFeed(CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
|
||||
feed2 := db.CreateFeed(CreateFeedParams{Title: "feed 2", Link: "http://example2.com", FeedLink: "http://example2.com/feed.xml"})
|
||||
feeds := db.ListFeeds()
|
||||
if !reflect.DeepEqual(feeds, []Feed{*feed1, *feed2}) {
|
||||
t.Fatalf("invalid feed list: %#v", feeds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFeed(t *testing.T) {
|
||||
db := testDB()
|
||||
feed1 := db.CreateFeed(CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
|
||||
folder := db.CreateFolder("test")
|
||||
icon := []byte("icon")
|
||||
|
||||
title := "newtitle"
|
||||
db.UpdateFeed(feed1.Id, UpdateFeedParams{
|
||||
Title: &title,
|
||||
FolderID: SetNullable(&folder.Id),
|
||||
Icon: SetNullable(&icon),
|
||||
})
|
||||
|
||||
feed2 := db.GetFeed(feed1.Id)
|
||||
if feed2.Title != "newtitle" {
|
||||
t.Error("invalid title")
|
||||
}
|
||||
if feed2.FolderId == nil || *feed2.FolderId != folder.Id {
|
||||
t.Error("invalid folder")
|
||||
}
|
||||
if !feed2.HasIcon || string(*feed2.Icon) != "icon" {
|
||||
t.Error("invalid icon")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteFeed(t *testing.T) {
|
||||
db := testDB()
|
||||
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
|
||||
|
||||
if db.DeleteFeed(100500) {
|
||||
t.Error("cannot delete what does not exist")
|
||||
}
|
||||
|
||||
if !db.DeleteFeed(feed1.Id) {
|
||||
t.Fatal("did not delete existing feed")
|
||||
}
|
||||
if db.GetFeed(feed1.Id) != nil {
|
||||
t.Fatal("feed still exists")
|
||||
}
|
||||
}
|
||||
119
src/storage/sqlite/feedstate.go
Normal file
119
src/storage/sqlite/feedstate.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
)
|
||||
|
||||
type FeedState struct {
|
||||
FeedID int64
|
||||
LastRefreshed time.Time
|
||||
LastError string
|
||||
HTTPLastModified string
|
||||
HTTPEtag string
|
||||
}
|
||||
|
||||
func (s *Storage) ListFeedStates() ([]FeedState, error) {
|
||||
rows, err := s.db.Query(`
|
||||
select
|
||||
feed_id
|
||||
, last_refreshed
|
||||
, last_error
|
||||
, http_lmod
|
||||
, http_etag
|
||||
from feed_states
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
states := make([]FeedState, 0)
|
||||
for rows.Next() {
|
||||
var state FeedState
|
||||
err := rows.Scan(
|
||||
&state.FeedID,
|
||||
&state.LastRefreshed,
|
||||
&state.LastError,
|
||||
&state.HTTPLastModified,
|
||||
&state.HTTPEtag,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
states = append(states, state)
|
||||
}
|
||||
return states, nil
|
||||
}
|
||||
|
||||
func (s *Storage) GetFeedState(feedID int64) (*FeedState, error) {
|
||||
var state FeedState
|
||||
err := s.db.QueryRow(`
|
||||
select
|
||||
feed_id
|
||||
, last_refreshed
|
||||
, last_error
|
||||
, http_lmod
|
||||
, http_etag
|
||||
from feed_states where feed_id = :id
|
||||
`, sql.Named("id", feedID)).Scan(
|
||||
&state.FeedID,
|
||||
&state.LastRefreshed,
|
||||
&state.LastError,
|
||||
&state.HTTPLastModified,
|
||||
&state.HTTPEtag,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &state, nil
|
||||
}
|
||||
|
||||
type UpdateFeedStateParams struct {
|
||||
LastRefreshed *time.Time
|
||||
LastError *string
|
||||
HTTPLastModified *string
|
||||
HTTPEtag *string
|
||||
}
|
||||
|
||||
func (s *Storage) UpdateFeedState(feedID int64, params UpdateFeedStateParams) (bool, error) {
|
||||
lastError := params.LastError
|
||||
if lastError != nil && *lastError == "" {
|
||||
lastError = nil
|
||||
}
|
||||
|
||||
_, err := s.db.Exec(`
|
||||
insert into feed_states (
|
||||
feed_id
|
||||
, last_refreshed
|
||||
, last_error
|
||||
, http_lmod
|
||||
, http_etag
|
||||
)
|
||||
values (
|
||||
:id
|
||||
, coalesce(:last_refreshed, 0)
|
||||
, coalesce(:last_error, '')
|
||||
, coalesce(:http_lmod, '')
|
||||
, coalesce(:http_etag, '')
|
||||
)
|
||||
on conflict (feed_id) do update set
|
||||
last_refreshed = coalesce(:last_refreshed, last_refreshed),
|
||||
last_error = coalesce(:last_error, last_error),
|
||||
http_lmod = coalesce(:http_lmod, http_lmod),
|
||||
http_etag = coalesce(:http_etag, http_etag)
|
||||
`,
|
||||
sql.Named("id", feedID),
|
||||
sql.Named("last_refreshed", params.LastRefreshed),
|
||||
sql.Named("last_error", params.LastError),
|
||||
sql.Named("http_lmod", params.HTTPLastModified),
|
||||
sql.Named("http_etag", params.HTTPEtag),
|
||||
)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
129
src/storage/sqlite/feedstate_test.go
Normal file
129
src/storage/sqlite/feedstate_test.go
Normal file
@@ -0,0 +1,129 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestUpdateFeedState_Full(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
errMsg := "error"
|
||||
lmod := "today"
|
||||
etag := "v1"
|
||||
|
||||
ok, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
|
||||
LastRefreshed: &now,
|
||||
LastError: &errMsg,
|
||||
HTTPLastModified: &lmod,
|
||||
HTTPEtag: &etag,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !ok {
|
||||
t.Error("expected true")
|
||||
}
|
||||
|
||||
state, err := s.GetFeedState(f.Id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state == nil {
|
||||
t.Fatal("expected state, got nil")
|
||||
}
|
||||
if !state.LastRefreshed.Equal(now) {
|
||||
t.Errorf("expected %v, got %v", now, state.LastRefreshed)
|
||||
}
|
||||
if state.LastError != errMsg {
|
||||
t.Errorf("expected %s, got %v", errMsg, state.LastError)
|
||||
}
|
||||
if state.HTTPLastModified != lmod {
|
||||
t.Errorf("expected %s, got %s", lmod, state.HTTPLastModified)
|
||||
}
|
||||
if state.HTTPEtag != etag {
|
||||
t.Errorf("expected %s, got %s", etag, state.HTTPEtag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFeedState_Partial(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
|
||||
etag := "v1"
|
||||
s.UpdateFeedState(f.Id, UpdateFeedStateParams{HTTPEtag: &etag})
|
||||
|
||||
newErr := "new error"
|
||||
_, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
|
||||
LastError: &newErr,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
state, err := s.GetFeedState(f.Id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.LastError != newErr {
|
||||
t.Errorf("expected %s, got %v", newErr, state.LastError)
|
||||
}
|
||||
if state.HTTPEtag != etag {
|
||||
t.Errorf("etag should be unchanged, got %s", state.HTTPEtag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateFeedState_ClearError(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
|
||||
errMsg := "error"
|
||||
s.UpdateFeedState(f.Id, UpdateFeedStateParams{LastError: &errMsg})
|
||||
|
||||
empty := ""
|
||||
_, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
|
||||
LastError: &empty,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
state, err := s.GetFeedState(f.Id)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if state.LastError != "" {
|
||||
t.Errorf("expected empty error string, got %v", state.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFeedStates(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
f1 := s.CreateFeed(CreateFeedParams{Title: "F1", FeedLink: "L1"})
|
||||
f2 := s.CreateFeed(CreateFeedParams{Title: "F2", FeedLink: "L2"})
|
||||
|
||||
errMsg := "fail"
|
||||
s.UpdateFeedState(f1.Id, UpdateFeedStateParams{LastError: &errMsg})
|
||||
s.UpdateFeedState(f2.Id, UpdateFeedStateParams{HTTPEtag: ptr("e")})
|
||||
|
||||
states, err := s.ListFeedStates()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(states) != 2 {
|
||||
t.Errorf("expected 2 states, got %d", len(states))
|
||||
}
|
||||
}
|
||||
|
||||
func ptr[T any](v T) *T {
|
||||
return &v
|
||||
}
|
||||
85
src/storage/sqlite/folder.go
Normal file
85
src/storage/sqlite/folder.go
Normal file
@@ -0,0 +1,85 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
)
|
||||
|
||||
type Folder struct {
|
||||
Id int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
IsExpanded bool `json:"is_expanded"`
|
||||
}
|
||||
|
||||
func (s *Storage) CreateFolder(title string) *Folder {
|
||||
expanded := true
|
||||
row := s.db.QueryRow(`
|
||||
insert into folders (title, is_expanded) values (:title, :is_expanded)
|
||||
on conflict (title) do update set title = :title
|
||||
returning id`,
|
||||
sql.Named("title", title),
|
||||
sql.Named("is_expanded", expanded),
|
||||
)
|
||||
var id int64
|
||||
err := row.Scan(&id)
|
||||
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return nil
|
||||
}
|
||||
return &Folder{Id: id, Title: title, IsExpanded: expanded}
|
||||
}
|
||||
|
||||
func (s *Storage) DeleteFolder(folderId int64) bool {
|
||||
_, err := s.db.Exec(`delete from folders where id = :id`, sql.Named("id", folderId))
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
type UpdateFolderParams struct {
|
||||
Title *string
|
||||
IsExpanded *bool
|
||||
}
|
||||
|
||||
func (s *Storage) UpdateFolder(folderId int64, params UpdateFolderParams) (bool, error) {
|
||||
_, err := s.db.Exec(`
|
||||
update folders set
|
||||
title = coalesce(:title, title),
|
||||
is_expanded = coalesce(:is_expanded, is_expanded)
|
||||
where id = :id
|
||||
`,
|
||||
sql.Named("id", folderId),
|
||||
sql.Named("title", params.Title),
|
||||
sql.Named("is_expanded", params.IsExpanded),
|
||||
)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Storage) ListFolders() []Folder {
|
||||
result := make([]Folder, 0)
|
||||
rows, err := s.db.Query(`
|
||||
select id, title, is_expanded
|
||||
from folders
|
||||
order by title collate nocase
|
||||
`)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
for rows.Next() {
|
||||
var f Folder
|
||||
err = rows.Scan(&f.Id, &f.Title, &f.IsExpanded)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
result = append(result, f)
|
||||
}
|
||||
return result
|
||||
}
|
||||
78
src/storage/sqlite/folder_test.go
Normal file
78
src/storage/sqlite/folder_test.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestUpdateFolder(t *testing.T) {
|
||||
db := testDB()
|
||||
folder := db.CreateFolder("old title")
|
||||
if folder.IsExpanded != true {
|
||||
t.Fatal("expected folder to be expanded by default")
|
||||
}
|
||||
|
||||
t.Run("rename only", func(t *testing.T) {
|
||||
newTitle := "new title"
|
||||
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
|
||||
Title: &newTitle,
|
||||
})
|
||||
if !ok || err != nil {
|
||||
t.Fatalf("UpdateFolder failed: %v", err)
|
||||
}
|
||||
|
||||
folders := db.ListFolders()
|
||||
if len(folders) != 1 || folders[0].Title != "new title" {
|
||||
t.Errorf("expected title to be updated, got %s", folders[0].Title)
|
||||
}
|
||||
if folders[0].IsExpanded != true {
|
||||
t.Error("expected expansion state to remain unchanged")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("toggle expanded only", func(t *testing.T) {
|
||||
isExpanded := false
|
||||
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
|
||||
IsExpanded: &isExpanded,
|
||||
})
|
||||
if !ok || err != nil {
|
||||
t.Fatalf("UpdateFolder failed: %v", err)
|
||||
}
|
||||
|
||||
folders := db.ListFolders()
|
||||
if len(folders) != 1 || folders[0].IsExpanded != false {
|
||||
t.Errorf("expected is_expanded to be false, got %v", folders[0].IsExpanded)
|
||||
}
|
||||
if folders[0].Title != "new title" {
|
||||
t.Error("expected title to remain unchanged")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("update both", func(t *testing.T) {
|
||||
bothTitle := "both"
|
||||
isExpanded := true
|
||||
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
|
||||
Title: &bothTitle,
|
||||
IsExpanded: &isExpanded,
|
||||
})
|
||||
if !ok || err != nil {
|
||||
t.Fatalf("UpdateFolder failed: %v", err)
|
||||
}
|
||||
|
||||
folders := db.ListFolders()
|
||||
if len(folders) != 1 || folders[0].Title != "both" || folders[0].IsExpanded != true {
|
||||
t.Errorf("expected both to be updated, got title=%s expanded=%v", folders[0].Title, folders[0].IsExpanded)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("update none", func(t *testing.T) {
|
||||
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{})
|
||||
if !ok || err != nil {
|
||||
t.Fatalf("UpdateFolder failed: %v", err)
|
||||
}
|
||||
|
||||
folders := db.ListFolders()
|
||||
if len(folders) != 1 || folders[0].Title != "both" || folders[0].IsExpanded != true {
|
||||
t.Errorf("expected no changes, got title=%s expanded=%v", folders[0].Title, folders[0].IsExpanded)
|
||||
}
|
||||
})
|
||||
}
|
||||
424
src/storage/sqlite/item.go
Normal file
424
src/storage/sqlite/item.go
Normal file
@@ -0,0 +1,424 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type ItemStatus int
|
||||
|
||||
const (
|
||||
UNREAD ItemStatus = 0
|
||||
READ ItemStatus = 1
|
||||
STARRED ItemStatus = 2
|
||||
)
|
||||
|
||||
var StatusRepresentations = map[ItemStatus]string{
|
||||
UNREAD: "unread",
|
||||
READ: "read",
|
||||
STARRED: "starred",
|
||||
}
|
||||
|
||||
var StatusValues = map[string]ItemStatus{
|
||||
"unread": UNREAD,
|
||||
"read": READ,
|
||||
"starred": STARRED,
|
||||
}
|
||||
|
||||
func (s ItemStatus) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(StatusRepresentations[s])
|
||||
}
|
||||
|
||||
func (s *ItemStatus) UnmarshalJSON(b []byte) error {
|
||||
var str string
|
||||
if err := json.Unmarshal(b, &str); err != nil {
|
||||
return err
|
||||
}
|
||||
*s = StatusValues[str]
|
||||
return nil
|
||||
}
|
||||
|
||||
type MediaLink struct {
|
||||
URL string `json:"url"`
|
||||
Type string `json:"type"`
|
||||
Description string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
type MediaLinks []MediaLink
|
||||
|
||||
func (m *MediaLinks) Scan(src any) error {
|
||||
switch data := src.(type) {
|
||||
case []byte:
|
||||
return json.Unmarshal(data, m)
|
||||
case string:
|
||||
return json.Unmarshal([]byte(data), m)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (m MediaLinks) Value() (driver.Value, error) {
|
||||
return json.Marshal(m)
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Id int64 `json:"id"`
|
||||
GUID string `json:"guid"`
|
||||
FeedId int64 `json:"feed_id"`
|
||||
Title string `json:"title"`
|
||||
Link string `json:"link"`
|
||||
Content string `json:"content,omitempty"`
|
||||
Date time.Time `json:"date"`
|
||||
Status ItemStatus `json:"status"`
|
||||
MediaLinks MediaLinks `json:"media_links"`
|
||||
}
|
||||
|
||||
type ItemFilter struct {
|
||||
FolderID *int64
|
||||
FeedID *int64
|
||||
Status *ItemStatus
|
||||
Search *string
|
||||
After *int64
|
||||
IDs *[]int64
|
||||
SinceID *int64
|
||||
MaxID *int64
|
||||
Before *time.Time
|
||||
}
|
||||
|
||||
type MarkFilter struct {
|
||||
FolderID *int64
|
||||
FeedID *int64
|
||||
|
||||
Before *time.Time
|
||||
}
|
||||
|
||||
type ItemList []Item
|
||||
|
||||
func (list ItemList) Len() int {
|
||||
return len(list)
|
||||
}
|
||||
|
||||
func (list ItemList) SortKey(i int) string {
|
||||
return list[i].Date.Format(time.RFC3339) + "::" + list[i].GUID
|
||||
}
|
||||
|
||||
func (list ItemList) Less(i, j int) bool {
|
||||
return list.SortKey(i) < list.SortKey(j)
|
||||
}
|
||||
|
||||
func (list ItemList) Swap(i, j int) {
|
||||
list[i], list[j] = list[j], list[i]
|
||||
}
|
||||
|
||||
func (s *Storage) CreateItems(items []Item) bool {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return false
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
itemsSorted := ItemList(items)
|
||||
sort.Sort(itemsSorted)
|
||||
|
||||
for _, item := range itemsSorted {
|
||||
_, err = tx.Exec(`
|
||||
insert into items (
|
||||
guid, feed_id, title, link, date,
|
||||
content, media_links,
|
||||
date_arrived, last_arrived, status
|
||||
)
|
||||
values (
|
||||
:guid, :feed_id, :title, :link, strftime('%Y-%m-%d %H:%M:%f', :date),
|
||||
:content, :media_links,
|
||||
:date_arrived, :last_arrived, :status
|
||||
)
|
||||
on conflict (feed_id, guid) do update set
|
||||
last_arrived = :last_arrived`,
|
||||
sql.Named("guid", item.GUID),
|
||||
sql.Named("feed_id", item.FeedId),
|
||||
sql.Named("title", item.Title),
|
||||
sql.Named("link", item.Link),
|
||||
sql.Named("date", item.Date),
|
||||
sql.Named("content", item.Content),
|
||||
sql.Named("media_links", item.MediaLinks),
|
||||
sql.Named("date_arrived", now),
|
||||
sql.Named("last_arrived", now),
|
||||
sql.Named("status", UNREAD),
|
||||
)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
if err = tx.Rollback(); err != nil {
|
||||
log.Print(err)
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
log.Print(err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []any) {
|
||||
cond := make([]string, 0)
|
||||
args := make([]any, 0)
|
||||
if filter.FolderID != nil {
|
||||
cond = append(cond, "i.feed_id in (select id from feeds where folder_id = :folder_id)")
|
||||
args = append(args, sql.Named("folder_id", *filter.FolderID))
|
||||
}
|
||||
if filter.FeedID != nil {
|
||||
cond = append(cond, "i.feed_id = :feed_id")
|
||||
args = append(args, sql.Named("feed_id", *filter.FeedID))
|
||||
}
|
||||
if filter.Status != nil {
|
||||
cond = append(cond, "i.status = :status")
|
||||
args = append(args, sql.Named("status", *filter.Status))
|
||||
}
|
||||
if filter.Search != nil {
|
||||
words := strings.Fields(*filter.Search)
|
||||
terms := make([]string, len(words))
|
||||
for idx, word := range words {
|
||||
terms[idx] = word + "*"
|
||||
}
|
||||
|
||||
cond = append(
|
||||
cond,
|
||||
"i.id in (select rowid as id from search where search match :search)",
|
||||
)
|
||||
args = append(args, sql.Named("search", strings.Join(terms, " ")))
|
||||
}
|
||||
if filter.After != nil {
|
||||
compare := ">"
|
||||
if newestFirst {
|
||||
compare = "<"
|
||||
}
|
||||
cond = append(
|
||||
cond,
|
||||
fmt.Sprintf(
|
||||
"(i.date, i.id) %s (select date, id from items where id = :after_id)",
|
||||
compare,
|
||||
),
|
||||
)
|
||||
args = append(args, sql.Named("after_id", *filter.After))
|
||||
}
|
||||
if filter.IDs != nil && len(*filter.IDs) > 0 {
|
||||
qmarks := make([]string, len(*filter.IDs))
|
||||
for i, id := range *filter.IDs {
|
||||
name := fmt.Sprintf("id%d", i)
|
||||
qmarks[i] = ":" + name
|
||||
args = append(args, sql.Named(name, id))
|
||||
}
|
||||
cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")")
|
||||
}
|
||||
if filter.SinceID != nil {
|
||||
cond = append(cond, "i.id > :since_id")
|
||||
args = append(args, sql.Named("since_id", filter.SinceID))
|
||||
}
|
||||
if filter.MaxID != nil {
|
||||
cond = append(cond, "i.id < :max_id")
|
||||
args = append(args, sql.Named("max_id", filter.MaxID))
|
||||
}
|
||||
if filter.Before != nil {
|
||||
cond = append(cond, "i.date < :before")
|
||||
args = append(args, sql.Named("before", filter.Before))
|
||||
}
|
||||
|
||||
predicate := "1"
|
||||
if len(cond) > 0 {
|
||||
predicate = strings.Join(cond, " and ")
|
||||
}
|
||||
|
||||
return predicate, args
|
||||
}
|
||||
|
||||
func (s *Storage) CountItems() int {
|
||||
var count int
|
||||
err := s.db.QueryRow(`select count(*) from items`).Scan(&count)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return 0
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (s *Storage) ListItems(
|
||||
filter ItemFilter,
|
||||
limit int,
|
||||
newestFirst bool,
|
||||
withContent bool,
|
||||
) []Item {
|
||||
predicate, args := listQueryPredicate(filter, newestFirst)
|
||||
result := make([]Item, 0)
|
||||
|
||||
order := "date desc, id desc"
|
||||
if !newestFirst {
|
||||
order = "date asc, id asc"
|
||||
}
|
||||
if filter.IDs != nil || filter.SinceID != nil {
|
||||
order = "i.id asc"
|
||||
}
|
||||
if filter.MaxID != nil {
|
||||
order = "i.id desc"
|
||||
}
|
||||
|
||||
selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.media_links"
|
||||
if withContent {
|
||||
selectCols += ", i.content"
|
||||
} else {
|
||||
selectCols += ", '' as content"
|
||||
}
|
||||
query := fmt.Sprintf(`
|
||||
select %s
|
||||
from items i
|
||||
where %s
|
||||
order by %s
|
||||
limit %d
|
||||
`, selectCols, predicate, order, limit)
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
for rows.Next() {
|
||||
var x Item
|
||||
err = rows.Scan(
|
||||
&x.Id, &x.GUID, &x.FeedId,
|
||||
&x.Title, &x.Link, &x.Date,
|
||||
&x.Status, &x.MediaLinks, &x.Content,
|
||||
)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
result = append(result, x)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *Storage) GetItem(id int64) *Item {
|
||||
i := &Item{}
|
||||
err := s.db.QueryRow(`
|
||||
select
|
||||
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
|
||||
i.date, i.status, i.media_links
|
||||
from items i
|
||||
where i.id = :id
|
||||
`, sql.Named("id", id)).Scan(
|
||||
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
|
||||
&i.Date, &i.Status, &i.MediaLinks,
|
||||
)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return nil
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
|
||||
_, err := s.db.Exec(`update items set status = :status where id = :id`,
|
||||
sql.Named("status", status),
|
||||
sql.Named("id", item_id),
|
||||
)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
|
||||
predicate, args := listQueryPredicate(ItemFilter{
|
||||
FolderID: filter.FolderID,
|
||||
FeedID: filter.FeedID,
|
||||
Before: filter.Before,
|
||||
}, false)
|
||||
query := fmt.Sprintf(`
|
||||
update items as i set status = %d
|
||||
where %s and i.status != %d
|
||||
`, READ, predicate, STARRED)
|
||||
_, err := s.db.Exec(query, args...)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
return err == nil
|
||||
}
|
||||
|
||||
type FeedStat struct {
|
||||
FeedId int64 `json:"feed_id"`
|
||||
UnreadCount int64 `json:"unread"`
|
||||
StarredCount int64 `json:"starred"`
|
||||
}
|
||||
|
||||
func (s *Storage) FeedStats() []FeedStat {
|
||||
result := make([]FeedStat, 0)
|
||||
rows, err := s.db.Query(fmt.Sprintf(`
|
||||
select
|
||||
feed_id,
|
||||
sum(case status when %d then 1 else 0 end),
|
||||
sum(case status when %d then 1 else 0 end)
|
||||
from items
|
||||
group by feed_id
|
||||
`, UNREAD, STARRED))
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
for rows.Next() {
|
||||
stat := FeedStat{}
|
||||
rows.Scan(&stat.FeedId, &stat.UnreadCount, &stat.StarredCount)
|
||||
result = append(result, stat)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
var (
|
||||
itemsKeepSize = 50
|
||||
itemsKeepDays = 90
|
||||
)
|
||||
|
||||
// Delete old articles from the database to cleanup space.
|
||||
//
|
||||
// The rules:
|
||||
// - Never delete starred entries.
|
||||
// - Keep at least 50 latest items for each feed.
|
||||
// - Delete entries older than 90 days relative to the latest arrived item in the same feed.
|
||||
func (s *Storage) DeleteOldItems() {
|
||||
result, err := s.db.Exec(`
|
||||
delete from items
|
||||
where id in (
|
||||
select id
|
||||
from (
|
||||
select
|
||||
id,
|
||||
row_number() over (partition by feed_id order by date desc) as rn,
|
||||
last_arrived,
|
||||
max(last_arrived) over (partition by feed_id) as max_la
|
||||
from items
|
||||
where status != :starred_status
|
||||
)
|
||||
where rn > :keep_size
|
||||
and last_arrived < datetime(max_la, :keep_days_limit)
|
||||
)`,
|
||||
sql.Named("starred_status", STARRED),
|
||||
sql.Named("keep_size", itemsKeepSize),
|
||||
sql.Named("keep_days_limit", fmt.Sprintf("-%d days", itemsKeepDays)),
|
||||
)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return
|
||||
}
|
||||
numDeleted, err := result.RowsAffected()
|
||||
if err == nil && numDeleted > 0 {
|
||||
log.Printf("Deleted %d old items", numDeleted)
|
||||
|
||||
if _, err := s.db.Exec("vacuum"); err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
515
src/storage/sqlite/item_test.go
Normal file
515
src/storage/sqlite/item_test.go
Normal file
@@ -0,0 +1,515 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
/*
|
||||
- folder1
|
||||
- feed11
|
||||
- item111 (unread)
|
||||
- item112 (read)
|
||||
- item113 (starred)
|
||||
- feed12
|
||||
- item121 (unread)
|
||||
- item122 (read)
|
||||
- folder2
|
||||
- feed21
|
||||
- item211 (read)
|
||||
- item212 (starred)
|
||||
- feed01
|
||||
- item011 (unread)
|
||||
- item012 (read)
|
||||
- item013 (starred)
|
||||
*/
|
||||
|
||||
type testItemScope struct {
|
||||
feed11, feed12 *Feed
|
||||
feed21, feed01 *Feed
|
||||
folder1, folder2 *Folder
|
||||
}
|
||||
|
||||
func testItemsSetup(db *Storage) testItemScope {
|
||||
folder1 := db.CreateFolder("folder1")
|
||||
folder2 := db.CreateFolder("folder2")
|
||||
|
||||
feed11 := db.CreateFeed(CreateFeedParams{Title: "feed11", FeedLink: "http://test.com/feed11.xml", FolderID: &folder1.Id})
|
||||
feed12 := db.CreateFeed(CreateFeedParams{Title: "feed12", FeedLink: "http://test.com/feed12.xml", FolderID: &folder1.Id})
|
||||
feed21 := db.CreateFeed(CreateFeedParams{Title: "feed21", FeedLink: "http://test.com/feed21.xml", FolderID: &folder2.Id})
|
||||
feed01 := db.CreateFeed(CreateFeedParams{Title: "feed01", FeedLink: "http://test.com/feed01.xml"})
|
||||
|
||||
now := time.Now()
|
||||
db.CreateItems([]Item{
|
||||
// feed11
|
||||
{GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)},
|
||||
{
|
||||
GUID: "item112",
|
||||
FeedId: feed11.Id,
|
||||
Title: "title112",
|
||||
Date: now.Add(time.Hour * 24 * 2),
|
||||
}, // read
|
||||
{
|
||||
GUID: "item113",
|
||||
FeedId: feed11.Id,
|
||||
Title: "title113",
|
||||
Date: now.Add(time.Hour * 24 * 3),
|
||||
}, // starred
|
||||
// feed12
|
||||
{GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)},
|
||||
{
|
||||
GUID: "item122",
|
||||
FeedId: feed12.Id,
|
||||
Title: "title122",
|
||||
Date: now.Add(time.Hour * 24 * 5),
|
||||
}, // read
|
||||
// feed21
|
||||
{
|
||||
GUID: "item211",
|
||||
FeedId: feed21.Id,
|
||||
Title: "title211",
|
||||
Date: now.Add(time.Hour * 24 * 6),
|
||||
}, // read
|
||||
{
|
||||
GUID: "item212",
|
||||
FeedId: feed21.Id,
|
||||
Title: "title212",
|
||||
Date: now.Add(time.Hour * 24 * 7),
|
||||
}, // starred
|
||||
// feed01
|
||||
{GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)},
|
||||
{
|
||||
GUID: "item012",
|
||||
FeedId: feed01.Id,
|
||||
Title: "title012",
|
||||
Date: now.Add(time.Hour * 24 * 9),
|
||||
}, // read
|
||||
{
|
||||
GUID: "item013",
|
||||
FeedId: feed01.Id,
|
||||
Title: "title013",
|
||||
Date: now.Add(time.Hour * 24 * 10),
|
||||
}, // starred
|
||||
})
|
||||
db.db.Exec(
|
||||
`update items set status = :status where guid in ("item112", "item122", "item211", "item012")`,
|
||||
sql.Named("status", READ),
|
||||
)
|
||||
db.db.Exec(
|
||||
`update items set status = :status where guid in ("item113", "item212", "item013")`,
|
||||
sql.Named("status", STARRED),
|
||||
)
|
||||
|
||||
return testItemScope{
|
||||
feed11: feed11,
|
||||
feed12: feed12,
|
||||
feed21: feed21,
|
||||
feed01: feed01,
|
||||
folder1: folder1,
|
||||
folder2: folder2,
|
||||
}
|
||||
}
|
||||
|
||||
func getItem(db *Storage, guid string) *Item {
|
||||
i := &Item{}
|
||||
err := db.db.QueryRow(`
|
||||
select
|
||||
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
|
||||
i.date, i.status, i.media_links
|
||||
from items i
|
||||
where i.guid = :guid
|
||||
`, sql.Named("guid", guid)).Scan(
|
||||
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
|
||||
&i.Date, &i.Status, &i.MediaLinks,
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return i
|
||||
}
|
||||
|
||||
func getItemGuids(items []Item) []string {
|
||||
guids := make([]string, 0)
|
||||
for _, item := range items {
|
||||
guids = append(guids, item.GUID)
|
||||
}
|
||||
return guids
|
||||
}
|
||||
|
||||
func TestListItems(t *testing.T) {
|
||||
db := testDB()
|
||||
scope := testItemsSetup(db)
|
||||
|
||||
// filter by folder_id
|
||||
|
||||
have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
|
||||
want := []string{"item111", "item112", "item113", "item121", "item122"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
|
||||
want = []string{"item211", "item212"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
// filter by feed_id
|
||||
|
||||
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
|
||||
want = []string{"item111", "item112", "item113"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
|
||||
want = []string{"item011", "item012", "item013"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
// filter by status
|
||||
|
||||
var starred ItemStatus = STARRED
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
|
||||
want = []string{"item113", "item212", "item013"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
var unread ItemStatus = UNREAD
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
|
||||
want = []string{"item111", "item121", "item011"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
// limit
|
||||
|
||||
have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
|
||||
want = []string{"item111", "item112"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
// filter by search
|
||||
search1 := "title111"
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
|
||||
want = []string{"item111"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
// sort by date
|
||||
have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
|
||||
want = []string{"item013", "item012", "item011", "item212"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
func TestListItemsPaginated(t *testing.T) {
|
||||
db := testDB()
|
||||
testItemsSetup(db)
|
||||
|
||||
item012 := getItem(db, "item012")
|
||||
item121 := getItem(db, "item121")
|
||||
|
||||
// all, newest first
|
||||
have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
|
||||
want := []string{"item011", "item212", "item211"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
// unread, newest first
|
||||
unread := UNREAD
|
||||
have = getItemGuids(
|
||||
db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false),
|
||||
)
|
||||
want = []string{"item011", "item121", "item111"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
// starred, oldest first
|
||||
starred := STARRED
|
||||
have = getItemGuids(
|
||||
db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false),
|
||||
)
|
||||
want = []string{"item212", "item013"}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkItemsRead(t *testing.T) {
|
||||
// NOTE: starred items must not be marked as read
|
||||
var read ItemStatus = READ
|
||||
|
||||
db1 := testDB()
|
||||
testItemsSetup(db1)
|
||||
db1.MarkItemsRead(MarkFilter{})
|
||||
have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
|
||||
want := []string{
|
||||
"item111", "item112", "item121", "item122",
|
||||
"item211", "item011", "item012",
|
||||
}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
db2 := testDB()
|
||||
scope2 := testItemsSetup(db2)
|
||||
db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
|
||||
have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
|
||||
want = []string{
|
||||
"item111", "item112", "item121", "item122",
|
||||
"item211", "item012",
|
||||
}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
db3 := testDB()
|
||||
scope3 := testItemsSetup(db3)
|
||||
db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
|
||||
have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
|
||||
want = []string{
|
||||
"item111", "item112", "item122",
|
||||
"item211", "item012",
|
||||
}
|
||||
if !reflect.DeepEqual(have, want) {
|
||||
t.Logf("want: %#v", want)
|
||||
t.Logf("have: %#v", have)
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteOldItems(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
starred := STARRED
|
||||
|
||||
t.Run("keeps at least 50 items", func(t *testing.T) {
|
||||
db := testDB()
|
||||
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
|
||||
items := make([]Item, 100)
|
||||
for i := range 100 {
|
||||
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Hour * 24)}
|
||||
}
|
||||
db.CreateItems(items)
|
||||
|
||||
// // Set 1 recent (latest), 100 old (100 days ago)
|
||||
db.db.Exec(`update items set last_arrived = :la where guid = "99"`, sql.Named("la", now))
|
||||
db.db.Exec(`update items set last_arrived = :la where guid != "99"`, sql.Named("la", now.Add(-time.Hour*24*100)))
|
||||
|
||||
db.DeleteOldItems()
|
||||
var have int
|
||||
db.db.QueryRow("select count(*) from items where feed_id = ?", feed.Id).Scan(&have)
|
||||
if have != 50 {
|
||||
t.Errorf("expected 50 items, have %d", have)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("keeps all less than 90 days old", func(t *testing.T) {
|
||||
db := testDB()
|
||||
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
|
||||
items := make([]Item, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
|
||||
}
|
||||
db.CreateItems(items)
|
||||
|
||||
// Latest item at "now"
|
||||
// All others at 80 days ago (keep)
|
||||
db.db.Exec(`update items set last_arrived = :la where guid = "99"`, sql.Named("la", now))
|
||||
db.db.Exec(`update items set last_arrived = :la where guid != "99"`, sql.Named("la", now.Add(-time.Hour*24*80)))
|
||||
|
||||
db.DeleteOldItems()
|
||||
var have int
|
||||
db.db.QueryRow("select count(*) from items where feed_id = ?", feed.Id).Scan(&have)
|
||||
if have != 100 {
|
||||
t.Errorf("expected 100 items, have %d", have)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("keeps starred", func(t *testing.T) {
|
||||
db := testDB()
|
||||
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
|
||||
items := make([]Item, 100)
|
||||
for i := 0; i < 100; i++ {
|
||||
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
|
||||
}
|
||||
db.CreateItems(items)
|
||||
|
||||
// Set all to 100 days ago, except one recent
|
||||
db.db.Exec(`update items set last_arrived = :la`, sql.Named("la", now.Add(-time.Hour*24*100)))
|
||||
db.db.Exec(`update items set last_arrived = :la where guid = "99"`, sql.Named("la", now))
|
||||
// Star 10 old items that would otherwise be deleted (rn > 50 and old)
|
||||
db.db.Exec(`update items set status = :s where cast(guid as integer) < 10`, sql.Named("s", starred))
|
||||
|
||||
db.DeleteOldItems()
|
||||
var have int
|
||||
db.db.QueryRow("select count(*) from items where feed_id = ?", feed.Id).Scan(&have)
|
||||
// 50 (limit) + 10 (starred) = 60 items should remain.
|
||||
if have != 60 {
|
||||
t.Errorf("expected 60 items, have %d", have)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
func TestCreateItemsLastArrived(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
db := testDB()
|
||||
defer db.db.Close()
|
||||
feed := db.CreateFeed(CreateFeedParams{Title: "test feed", FeedLink: "http://example.com/feed"})
|
||||
|
||||
item := Item{
|
||||
GUID: "item1",
|
||||
FeedId: feed.Id,
|
||||
Title: "Title 1",
|
||||
Date: time.Now(),
|
||||
}
|
||||
|
||||
// 1. Initial creation
|
||||
db.CreateItems([]Item{item})
|
||||
|
||||
var lastArrived1 time.Time
|
||||
err := db.db.QueryRow("select last_arrived from items where guid = ?", item.GUID).Scan(&lastArrived1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
time.Sleep(time.Second * 10)
|
||||
|
||||
// 2. Update on conflict
|
||||
db.CreateItems([]Item{item})
|
||||
|
||||
var lastArrived2 time.Time
|
||||
err = db.db.QueryRow("select last_arrived from items where guid = ?", item.GUID).Scan(&lastArrived2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if !lastArrived2.After(lastArrived1) {
|
||||
t.Errorf("expected last_arrived to be updated. old: %v, new: %v", lastArrived1, lastArrived2)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestSearch(t *testing.T) {
|
||||
db := testDB()
|
||||
defer db.Close()
|
||||
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
|
||||
|
||||
db.CreateItems([]Item{
|
||||
{
|
||||
GUID: "i1",
|
||||
FeedId: feed.Id,
|
||||
Title: "Hello World",
|
||||
Content: "This is a <b>test</b> of the <i>emergency</i> broadcast system.",
|
||||
},
|
||||
{
|
||||
GUID: "i2",
|
||||
FeedId: feed.Id,
|
||||
Title: "FTS5 Unicode",
|
||||
Content: "Unicode support with characters like: Привет, 世界, 🚀",
|
||||
},
|
||||
{
|
||||
GUID: "i3",
|
||||
FeedId: feed.Id,
|
||||
Title: "Hidden Tag",
|
||||
Content: `<div class="secret-class">Don't find me by my class name</div>`,
|
||||
},
|
||||
})
|
||||
|
||||
// 1. Basic search
|
||||
s1 := "emergency"
|
||||
have := getItemGuids(db.ListItems(ItemFilter{Search: &s1}, 10, true, false))
|
||||
if !reflect.DeepEqual(have, []string{"i1"}) {
|
||||
t.Errorf("basic search failed: expected [i1], got %v", have)
|
||||
}
|
||||
|
||||
// 2. HTML stripping: Should find text, but NOT the tags
|
||||
s2 := "test"
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &s2}, 10, true, false))
|
||||
if !reflect.DeepEqual(have, []string{"i1"}) {
|
||||
t.Errorf("html text search failed: expected [i1], got %v", have)
|
||||
}
|
||||
|
||||
s3 := "secret-class"
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &s3}, 10, true, false))
|
||||
if len(have) > 0 {
|
||||
t.Errorf("html tag search should have failed but found: %v", have)
|
||||
}
|
||||
|
||||
// 3. Multi-word (AND)
|
||||
s4 := "broadcast system"
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &s4}, 10, true, false))
|
||||
if !reflect.DeepEqual(have, []string{"i1"}) {
|
||||
t.Errorf("multi-word search failed: expected [i1], got %v", have)
|
||||
}
|
||||
|
||||
// 4. Unicode
|
||||
s5 := "Привет"
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &s5}, 10, true, false))
|
||||
if !reflect.DeepEqual(have, []string{"i2"}) {
|
||||
t.Errorf("unicode search failed: expected [i2], got %v", have)
|
||||
}
|
||||
|
||||
s6 := "世界"
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &s6}, 10, true, false))
|
||||
if !reflect.DeepEqual(have, []string{"i2"}) {
|
||||
t.Errorf("unicode search (CJK) failed: expected [i2], got %v", have)
|
||||
}
|
||||
|
||||
// 5. Trigger: Update
|
||||
db.db.Exec("update items set title = 'Updated Title' where guid = 'i1'")
|
||||
s7 := "Updated"
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &s7}, 10, true, false))
|
||||
if !reflect.DeepEqual(have, []string{"i1"}) {
|
||||
t.Errorf("update trigger failed: expected [i1], got %v", have)
|
||||
}
|
||||
|
||||
// 6. Trigger: Delete
|
||||
db.db.Exec("delete from items where guid = 'i1'")
|
||||
have = getItemGuids(db.ListItems(ItemFilter{Search: &s7}, 10, true, false))
|
||||
if len(have) > 0 {
|
||||
t.Errorf("delete trigger failed: found deleted item: %v", have)
|
||||
}
|
||||
}
|
||||
422
src/storage/sqlite/migration.go
Normal file
422
src/storage/sqlite/migration.go
Normal file
@@ -0,0 +1,422 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
var migrations = []func(*sql.Tx) error{
|
||||
m01_initial,
|
||||
m02_feed_states_and_errors,
|
||||
m03_on_delete_actions,
|
||||
m04_item_podcasturl,
|
||||
m05_move_description_to_content,
|
||||
m06_fill_missing_dates,
|
||||
m07_add_feed_size,
|
||||
m08_normalize_datetime,
|
||||
m09_change_item_index,
|
||||
m10_add_item_medialinks,
|
||||
m11_add_item_last_arrived,
|
||||
m12_remove_feed_sizes,
|
||||
m13_consolidate_feed_states,
|
||||
m14_upgrade_fts5,
|
||||
}
|
||||
|
||||
var maxVersion = int64(len(migrations))
|
||||
|
||||
func migrate(db *sql.DB) error {
|
||||
var version int64
|
||||
if err := db.QueryRow("pragma user_version").Scan(&version); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if version >= maxVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("db version is %d. migrating to %d", version, maxVersion)
|
||||
|
||||
for v := version + 1; v <= maxVersion; v++ {
|
||||
// Migrations altering schema using a sequence of steps due to SQLite limitations.
|
||||
// Must come with `pragma foreign_key_check` at the end. See:
|
||||
// "Making Other Kinds Of Table Schema Changes"
|
||||
// https://www.sqlite.org/lang_altertable.html
|
||||
trickyAlteration := (v == 3)
|
||||
|
||||
log.Printf("[migration:%d] starting", v)
|
||||
|
||||
if trickyAlteration {
|
||||
db.Exec("pragma foreign_keys=off;")
|
||||
}
|
||||
|
||||
err := migrateVersion(v, db)
|
||||
|
||||
if trickyAlteration {
|
||||
db.Exec("pragma foreign_keys=on;")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("[migration:%d] done", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateVersion(v int64, db *sql.DB) error {
|
||||
var err error
|
||||
var tx *sql.Tx
|
||||
migratefunc := migrations[v-1]
|
||||
if tx, err = db.Begin(); err != nil {
|
||||
log.Printf("[migration:%d] failed to start transaction", v)
|
||||
return err
|
||||
}
|
||||
if err = migratefunc(tx); err != nil {
|
||||
log.Printf("[migration:%d] failed to migrate", v)
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(fmt.Sprintf("pragma user_version = %d", v)); err != nil {
|
||||
log.Printf("[migration:%d] failed to bump version", v)
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
log.Printf("[migration:%d] failed to commit changes", v)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func m01_initial(tx *sql.Tx) error {
|
||||
sql := `
|
||||
create table if not exists folders (
|
||||
id integer primary key autoincrement,
|
||||
title text not null,
|
||||
is_expanded boolean not null default false
|
||||
);
|
||||
|
||||
create unique index if not exists idx_folder_title on folders(title);
|
||||
|
||||
create table if not exists feeds (
|
||||
id integer primary key autoincrement,
|
||||
folder_id references folders(id),
|
||||
title text not null,
|
||||
description text,
|
||||
link text,
|
||||
feed_link text not null,
|
||||
icon blob
|
||||
);
|
||||
|
||||
create index if not exists idx_feed_folder_id on feeds(folder_id);
|
||||
create unique index if not exists idx_feed_feed_link on feeds(feed_link);
|
||||
|
||||
create table if not exists items (
|
||||
id integer primary key autoincrement,
|
||||
guid string not null,
|
||||
feed_id references feeds(id),
|
||||
title text,
|
||||
link text,
|
||||
description text,
|
||||
content text,
|
||||
author text,
|
||||
date datetime,
|
||||
date_updated datetime,
|
||||
date_arrived datetime,
|
||||
status integer,
|
||||
image text,
|
||||
search_rowid integer
|
||||
);
|
||||
|
||||
create index if not exists idx_item_feed_id on items(feed_id);
|
||||
create index if not exists idx_item_status on items(status);
|
||||
create index if not exists idx_item_search_rowid on items(search_rowid);
|
||||
create unique index if not exists idx_item_guid on items(feed_id, guid);
|
||||
|
||||
create table if not exists settings (
|
||||
key string primary key,
|
||||
val blob
|
||||
);
|
||||
|
||||
create virtual table if not exists search using fts4(title, description, content);
|
||||
|
||||
create trigger if not exists del_item_search after delete on items begin
|
||||
delete from search where rowid = old.search_rowid;
|
||||
end;
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m02_feed_states_and_errors(tx *sql.Tx) error {
|
||||
sql := `
|
||||
create table if not exists http_states (
|
||||
feed_id references feeds(id) unique,
|
||||
last_refreshed datetime not null,
|
||||
|
||||
-- http header fields --
|
||||
last_modified string not null,
|
||||
etag string not null
|
||||
);
|
||||
|
||||
create table if not exists feed_errors (
|
||||
feed_id references feeds(id) unique,
|
||||
error string
|
||||
);
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m03_on_delete_actions(tx *sql.Tx) error {
|
||||
sql := `
|
||||
-- 01. create altered tables
|
||||
create table if not exists new_feeds (
|
||||
id integer primary key autoincrement,
|
||||
folder_id references folders(id) on delete set null,
|
||||
title text not null,
|
||||
description text,
|
||||
link text,
|
||||
feed_link text not null,
|
||||
icon blob
|
||||
);
|
||||
create table if not exists new_items (
|
||||
id integer primary key autoincrement,
|
||||
guid string not null,
|
||||
feed_id references feeds(id) on delete cascade,
|
||||
title text,
|
||||
link text,
|
||||
description text,
|
||||
content text,
|
||||
author text,
|
||||
date datetime,
|
||||
date_updated datetime,
|
||||
date_arrived datetime,
|
||||
status integer,
|
||||
image text,
|
||||
search_rowid integer
|
||||
);
|
||||
create table if not exists new_http_states (
|
||||
feed_id references feeds(id) on delete cascade unique,
|
||||
last_refreshed datetime not null,
|
||||
last_modified string not null,
|
||||
etag string not null
|
||||
);
|
||||
create table if not exists new_feed_errors (
|
||||
feed_id references feeds(id) on delete cascade unique,
|
||||
error string
|
||||
);
|
||||
|
||||
-- 02. transfer data into new tables
|
||||
insert into new_feeds select * from feeds;
|
||||
insert into new_items select * from items;
|
||||
insert into new_http_states select * from http_states;
|
||||
insert into new_feed_errors select * from feed_errors;
|
||||
|
||||
-- 03. drop old tables
|
||||
drop table feeds;
|
||||
drop table items;
|
||||
drop table http_states;
|
||||
drop table feed_errors;
|
||||
|
||||
-- 04. rename new tables
|
||||
alter table new_feeds rename to feeds;
|
||||
alter table new_items rename to items;
|
||||
alter table new_http_states rename to http_states;
|
||||
alter table new_feed_errors rename to feed_errors;
|
||||
|
||||
-- 05. reconstruct indexes & triggers
|
||||
create index if not exists idx_feed_folder_id on feeds(folder_id);
|
||||
create unique index if not exists idx_feed_feed_link on feeds(feed_link);
|
||||
create index if not exists idx_item_feed_id on items(feed_id);
|
||||
create index if not exists idx_item_status on items(status);
|
||||
create index if not exists idx_item_search_rowid on items(search_rowid);
|
||||
create unique index if not exists idx_item_guid on items(feed_id, guid);
|
||||
create trigger if not exists del_item_search after delete on items begin
|
||||
delete from search where rowid = old.search_rowid;
|
||||
end;
|
||||
|
||||
-- 06. check consistency
|
||||
pragma foreign_key_check;
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m04_item_podcasturl(tx *sql.Tx) error {
|
||||
sql := `
|
||||
alter table items add column podcast_url text
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m05_move_description_to_content(tx *sql.Tx) error {
|
||||
sql := `
|
||||
update items set content=description
|
||||
where length(content) = 0 or content is null
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m06_fill_missing_dates(tx *sql.Tx) error {
|
||||
sql := `
|
||||
update items set date = 0 where date is null;
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m07_add_feed_size(tx *sql.Tx) error {
|
||||
sql := `
|
||||
create table if not exists feed_sizes (
|
||||
feed_id references feeds(id) on delete cascade unique,
|
||||
size integer not null default 0
|
||||
);
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m08_normalize_datetime(tx *sql.Tx) error {
|
||||
rows, err := tx.Query(`select id, date_arrived from items;`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var dateArrived time.Time
|
||||
err = rows.Scan(&id, &dateArrived)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(`update items set date_arrived = :date_arrived where id = :id;`,
|
||||
sql.Named("date_arrived", dateArrived.UTC()),
|
||||
sql.Named("id", id),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = tx.Exec(`update items set date = strftime('%Y-%m-%d %H:%M:%f', date);`)
|
||||
return err
|
||||
}
|
||||
|
||||
func m09_change_item_index(tx *sql.Tx) error {
|
||||
sql := `
|
||||
drop index if exists idx_item_status;
|
||||
create index if not exists idx_item__date_id_status on items(date,id,status);
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m10_add_item_medialinks(tx *sql.Tx) error {
|
||||
sql := `
|
||||
alter table items add column media_links json;
|
||||
update items set media_links = case
|
||||
when coalesce(image, '') != '' and coalesce(podcast_url, '') != ''
|
||||
then json_array(json_object('type', 'image', 'url', image), json_object('type', 'audio', 'url', podcast_url))
|
||||
|
||||
when coalesce(image, '') != ''
|
||||
then json_array(json_object('type', 'image', 'url', image))
|
||||
|
||||
when coalesce(podcast_url, '') != ''
|
||||
then json_array(json_object('type', 'audio', 'url', podcast_url))
|
||||
|
||||
else null
|
||||
end;
|
||||
alter table items drop column image;
|
||||
alter table items drop column podcast_url;
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m11_add_item_last_arrived(tx *sql.Tx) error {
|
||||
sql := `alter table items add column last_arrived datetime`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m12_remove_feed_sizes(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`drop table if exists feed_sizes`)
|
||||
return err
|
||||
}
|
||||
|
||||
func m13_consolidate_feed_states(tx *sql.Tx) error {
|
||||
sql := `
|
||||
create table feed_states (
|
||||
feed_id references feeds(id) on delete cascade unique
|
||||
, last_refreshed datetime not null default 0
|
||||
, last_error string not null default ''
|
||||
|
||||
, http_lmod string not null default ''
|
||||
, http_etag string not null default ''
|
||||
);
|
||||
|
||||
insert into feed_states (
|
||||
feed_id
|
||||
, last_refreshed
|
||||
, last_error
|
||||
, http_lmod
|
||||
, http_etag
|
||||
)
|
||||
select
|
||||
f.id
|
||||
, coalesce(h.last_refreshed, 0)
|
||||
, coalesce(e.error, '')
|
||||
, coalesce(h.last_modified, '')
|
||||
, coalesce(h.etag, '')
|
||||
from feeds f
|
||||
left join http_states h on f.id = h.feed_id
|
||||
left join feed_errors e on f.id = e.feed_id
|
||||
where h.feed_id is not null or e.feed_id is not null;
|
||||
|
||||
drop table http_states;
|
||||
drop table feed_errors;
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
|
||||
func m14_upgrade_fts5(tx *sql.Tx) error {
|
||||
sql := `
|
||||
-- 1. Drop old FTS4 table and trigger
|
||||
drop table if exists search;
|
||||
drop trigger if exists del_item_search;
|
||||
|
||||
-- 2. Remove search_rowid from items
|
||||
drop index if exists idx_item_search_rowid;
|
||||
alter table items drop column search_rowid;
|
||||
|
||||
-- 3. Create FTS5 virtual table
|
||||
create virtual table search using fts5(
|
||||
title, content,
|
||||
content='items',
|
||||
content_rowid='id',
|
||||
tokenize='unicode61'
|
||||
);
|
||||
|
||||
-- 4. Create triggers for automatic FTS sync
|
||||
create trigger items_ai after insert on items begin
|
||||
insert into search(rowid, title, content) values (new.id, new.title, strip_html(new.content));
|
||||
end;
|
||||
create trigger items_ad after delete on items begin
|
||||
insert into search(search, rowid, title, content) values('delete', old.id, old.title, strip_html(old.content));
|
||||
end;
|
||||
create trigger items_au after update on items begin
|
||||
insert into search(search, rowid, title, content) values('delete', old.id, old.title, strip_html(old.content));
|
||||
insert into search(rowid, title, content) values (new.id, new.title, strip_html(new.content));
|
||||
end;
|
||||
|
||||
-- 5. Populate FTS5 table with existing data
|
||||
insert into search(rowid, title, content) select id, title, strip_html(content) from items;
|
||||
`
|
||||
_, err := tx.Exec(sql)
|
||||
return err
|
||||
}
|
||||
171
src/storage/sqlite/settings.go
Normal file
171
src/storage/sqlite/settings.go
Normal file
@@ -0,0 +1,171 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"log"
|
||||
)
|
||||
|
||||
type Settings struct {
|
||||
Filter string `json:"filter"`
|
||||
Feed string `json:"feed"`
|
||||
FeedListWidth int `json:"feed_list_width"`
|
||||
ItemListWidth int `json:"item_list_width"`
|
||||
SortNewestFirst bool `json:"sort_newest_first"`
|
||||
ThemeName string `json:"theme_name"`
|
||||
ThemeFont string `json:"theme_font"`
|
||||
ThemeSize int `json:"theme_size"`
|
||||
RefreshRate int64 `json:"refresh_rate"`
|
||||
Language string `json:"language"`
|
||||
}
|
||||
|
||||
func (s Settings) Map() map[string]any {
|
||||
return map[string]any{
|
||||
"filter": s.Filter,
|
||||
"feed": s.Feed,
|
||||
"feed_list_width": s.FeedListWidth,
|
||||
"item_list_width": s.ItemListWidth,
|
||||
"sort_newest_first": s.SortNewestFirst,
|
||||
"theme_name": s.ThemeName,
|
||||
"theme_font": s.ThemeFont,
|
||||
"theme_size": s.ThemeSize,
|
||||
"refresh_rate": s.RefreshRate,
|
||||
"language": s.Language,
|
||||
}
|
||||
}
|
||||
|
||||
func settingsDefaults() Settings {
|
||||
return Settings{
|
||||
Filter: "",
|
||||
Feed: "",
|
||||
FeedListWidth: 300,
|
||||
ItemListWidth: 300,
|
||||
SortNewestFirst: true,
|
||||
ThemeName: "light",
|
||||
ThemeFont: "",
|
||||
ThemeSize: 1,
|
||||
RefreshRate: 0,
|
||||
Language: "en",
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Storage) GetSettings() Settings {
|
||||
result := settingsDefaults()
|
||||
rows, err := s.db.Query(`select key, val from settings;`)
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return result
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var key string
|
||||
var val []byte
|
||||
rows.Scan(&key, &val)
|
||||
|
||||
switch key {
|
||||
case "filter":
|
||||
json.Unmarshal(val, &result.Filter)
|
||||
case "feed":
|
||||
json.Unmarshal(val, &result.Feed)
|
||||
case "feed_list_width":
|
||||
json.Unmarshal(val, &result.FeedListWidth)
|
||||
case "item_list_width":
|
||||
json.Unmarshal(val, &result.ItemListWidth)
|
||||
case "sort_newest_first":
|
||||
json.Unmarshal(val, &result.SortNewestFirst)
|
||||
case "theme_name":
|
||||
json.Unmarshal(val, &result.ThemeName)
|
||||
case "theme_font":
|
||||
json.Unmarshal(val, &result.ThemeFont)
|
||||
case "theme_size":
|
||||
json.Unmarshal(val, &result.ThemeSize)
|
||||
case "refresh_rate":
|
||||
json.Unmarshal(val, &result.RefreshRate)
|
||||
case "language":
|
||||
json.Unmarshal(val, &result.Language)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type UpdateSettingsParams struct {
|
||||
Filter *string `json:"filter"`
|
||||
Feed *string `json:"feed"`
|
||||
FeedListWidth *int `json:"feed_list_width"`
|
||||
ItemListWidth *int `json:"item_list_width"`
|
||||
SortNewestFirst *bool `json:"sort_newest_first"`
|
||||
ThemeName *string `json:"theme_name"`
|
||||
ThemeFont *string `json:"theme_font"`
|
||||
ThemeSize *int `json:"theme_size"`
|
||||
RefreshRate *int64 `json:"refresh_rate"`
|
||||
Language *string `json:"language"`
|
||||
}
|
||||
|
||||
func (s *Storage) UpdateSettings(params UpdateSettingsParams) bool {
|
||||
tx, err := s.db.Begin()
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return false
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
update := func(key string, val any) error {
|
||||
valEncoded, err := json.Marshal(val)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(`
|
||||
insert into settings (key, val) values (:key, :val)
|
||||
on conflict (key) do update set val=:val`,
|
||||
sql.Named("key", key),
|
||||
sql.Named("val", valEncoded),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
var errs []error
|
||||
if params.Filter != nil {
|
||||
errs = append(errs, update("filter", *params.Filter))
|
||||
}
|
||||
if params.Feed != nil {
|
||||
errs = append(errs, update("feed", *params.Feed))
|
||||
}
|
||||
if params.FeedListWidth != nil {
|
||||
errs = append(errs, update("feed_list_width", *params.FeedListWidth))
|
||||
}
|
||||
if params.ItemListWidth != nil {
|
||||
errs = append(errs, update("item_list_width", *params.ItemListWidth))
|
||||
}
|
||||
if params.SortNewestFirst != nil {
|
||||
errs = append(errs, update("sort_newest_first", *params.SortNewestFirst))
|
||||
}
|
||||
if params.ThemeName != nil {
|
||||
errs = append(errs, update("theme_name", *params.ThemeName))
|
||||
}
|
||||
if params.ThemeFont != nil {
|
||||
errs = append(errs, update("theme_font", *params.ThemeFont))
|
||||
}
|
||||
if params.ThemeSize != nil {
|
||||
errs = append(errs, update("theme_size", *params.ThemeSize))
|
||||
}
|
||||
if params.RefreshRate != nil {
|
||||
errs = append(errs, update("refresh_rate", *params.RefreshRate))
|
||||
}
|
||||
if params.Language != nil {
|
||||
errs = append(errs, update("language", *params.Language))
|
||||
}
|
||||
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
log.Print(err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
log.Print(err)
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
150
src/storage/sqlite/settings_test.go
Normal file
150
src/storage/sqlite/settings_test.go
Normal file
@@ -0,0 +1,150 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSettingsDefaults(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
settings := s.GetSettings()
|
||||
defaults := settingsDefaults()
|
||||
|
||||
if !reflect.DeepEqual(settings, defaults) {
|
||||
t.Errorf("expected defaults %+v, got %+v", defaults, settings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateSettings(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
params := UpdateSettingsParams{
|
||||
ThemeName: ptr("night"),
|
||||
FeedListWidth: ptr(400),
|
||||
RefreshRate: ptr(int64(15)),
|
||||
}
|
||||
|
||||
if ok := s.UpdateSettings(params); !ok {
|
||||
t.Fatal("UpdateSettings failed")
|
||||
}
|
||||
|
||||
settings := s.GetSettings()
|
||||
|
||||
if settings.ThemeName != "night" {
|
||||
t.Errorf("expected theme_name night, got %s", settings.ThemeName)
|
||||
}
|
||||
if settings.FeedListWidth != 400 {
|
||||
t.Errorf("expected feed_list_width 400, got %d", settings.FeedListWidth)
|
||||
}
|
||||
if settings.RefreshRate != 15 {
|
||||
t.Errorf("expected refresh_rate 15, got %d", settings.RefreshRate)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetSettings(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
s.UpdateSettings(UpdateSettingsParams{Language: ptr("fr")})
|
||||
|
||||
settings := s.GetSettings()
|
||||
if settings.Language != "fr" {
|
||||
t.Errorf("expected fr, got %v", settings.Language)
|
||||
}
|
||||
if settings.ThemeName != "light" {
|
||||
t.Errorf("expected light, got %v", settings.ThemeName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSettingsExhaustive(t *testing.T) {
|
||||
s := testDB()
|
||||
defer s.Close()
|
||||
|
||||
settingsType := reflect.TypeOf(Settings{})
|
||||
paramsType := reflect.TypeOf(UpdateSettingsParams{})
|
||||
|
||||
settings := s.GetSettings()
|
||||
m := settings.Map()
|
||||
|
||||
for i := 0; i < settingsType.NumField(); i++ {
|
||||
field := settingsType.Field(i)
|
||||
jsonTag := field.Tag.Get("json")
|
||||
if jsonTag == "" {
|
||||
t.Errorf("Field %s missing json tag", field.Name)
|
||||
continue
|
||||
}
|
||||
// json tags might have options like "name,omitempty", take only the first part
|
||||
jsonKey := strings.Split(jsonTag, ",")[0]
|
||||
|
||||
// 1. Check Map()
|
||||
if _, ok := m[jsonKey]; !ok {
|
||||
t.Errorf("Key %q (from field %s) missing from Settings.Map()", jsonKey, field.Name)
|
||||
}
|
||||
|
||||
// 2. Check UpdateSettingsParams
|
||||
foundInParams := false
|
||||
for j := 0; j < paramsType.NumField(); j++ {
|
||||
pField := paramsType.Field(j)
|
||||
pJsonTag := strings.Split(pField.Tag.Get("json"), ",")[0]
|
||||
if pJsonTag == jsonKey {
|
||||
foundInParams = true
|
||||
// Also check it's a pointer
|
||||
if pField.Type.Kind() != reflect.Ptr {
|
||||
t.Errorf("Field %s in UpdateSettingsParams should be a pointer", pField.Name)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if !foundInParams {
|
||||
t.Errorf("Key %q (from field %s) missing from UpdateSettingsParams", jsonKey, field.Name)
|
||||
}
|
||||
|
||||
// 3. Test round-trip update
|
||||
// We'll create a new UpdateSettingsParams and set ONLY this field
|
||||
paramsValue := reflect.New(paramsType).Elem()
|
||||
for j := 0; j < paramsType.NumField(); j++ {
|
||||
pField := paramsType.Field(j)
|
||||
pJsonTag := strings.Split(pField.Tag.Get("json"), ",")[0]
|
||||
if pJsonTag == jsonKey {
|
||||
// Create a new value of the underlying type
|
||||
val := reflect.New(field.Type).Elem()
|
||||
switch field.Type.Kind() {
|
||||
case reflect.String:
|
||||
val.SetString("test_" + jsonKey)
|
||||
case reflect.Int, reflect.Int64:
|
||||
val.SetInt(42)
|
||||
case reflect.Bool:
|
||||
val.SetBool(false)
|
||||
}
|
||||
paramsValue.Field(j).Set(val.Addr())
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if ok := s.UpdateSettings(paramsValue.Interface().(UpdateSettingsParams)); !ok {
|
||||
t.Errorf("UpdateSettings failed for %q", jsonKey)
|
||||
}
|
||||
|
||||
updated := s.GetSettings()
|
||||
updatedValue := reflect.ValueOf(updated).Field(i)
|
||||
|
||||
switch field.Type.Kind() {
|
||||
case reflect.String:
|
||||
if updatedValue.String() != "test_"+jsonKey {
|
||||
t.Errorf("Round-trip failed for %q: expected %q, got %q (check UpdateSettings/GetSettings switch)", jsonKey, "test_"+jsonKey, updatedValue.String())
|
||||
}
|
||||
case reflect.Int, reflect.Int64:
|
||||
if updatedValue.Int() != 42 {
|
||||
t.Errorf("Round-trip failed for %q: expected 42, got %d (check UpdateSettings/GetSettings switch)", jsonKey, updatedValue.Int())
|
||||
}
|
||||
case reflect.Bool:
|
||||
if updatedValue.Bool() != false {
|
||||
t.Errorf("Round-trip failed for %q: expected false, got %v (check UpdateSettings/GetSettings switch)", jsonKey, updatedValue.Bool())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
53
src/storage/sqlite/storage.go
Normal file
53
src/storage/sqlite/storage.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/mattn/go-sqlite3"
|
||||
"github.com/nkanaev/yarr/src/content/htmlutil"
|
||||
)
|
||||
|
||||
func init() {
|
||||
sql.Register("sqlite3_yarr", &sqlite3.SQLiteDriver{
|
||||
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
|
||||
return conn.RegisterFunc("strip_html", htmlutil.ExtractText, true)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
type Storage struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
type Nullable[T any] struct {
|
||||
Set bool
|
||||
Value *T
|
||||
}
|
||||
|
||||
func SetNullable[T any](v *T) Nullable[T] {
|
||||
return Nullable[T]{Set: true, Value: v}
|
||||
}
|
||||
|
||||
func New(path string) (*Storage, error) {
|
||||
if pos := strings.IndexRune(path, '?'); pos == -1 {
|
||||
params := "_journal=WAL&_sync=NORMAL&_busy_timeout=5000&cache=shared"
|
||||
log.Printf("opening db with params: %s", params)
|
||||
path = path + "?" + params
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite3_yarr", path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = migrate(db); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Storage{db: db}, nil
|
||||
}
|
||||
|
||||
func (s *Storage) Close() error {
|
||||
return s.db.Close()
|
||||
}
|
||||
28
src/storage/sqlite/storage_test.go
Normal file
28
src/storage/sqlite/storage_test.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func testDB() *Storage {
|
||||
log.SetOutput(io.Discard)
|
||||
db, err := New(":memory:")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
log.SetOutput(os.Stderr)
|
||||
return db
|
||||
}
|
||||
|
||||
func TestStorage(t *testing.T) {
|
||||
db, err := New(":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if db == nil {
|
||||
t.Fatal("no db")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user