create sqlite package

This commit is contained in:
nkanaev
2026-06-07 23:14:16 +01:00
parent 14835660fb
commit f29ad0c20a
14 changed files with 2 additions and 1 deletions

157
src/storage/sqlite/feed.go Normal file
View File

@@ -0,0 +1,157 @@
package storage
import (
"database/sql"
"log"
)
type Feed struct {
Id int64 `json:"id"`
FolderId *int64 `json:"folder_id"`
Title string `json:"title"`
Description string `json:"description"`
Link string `json:"link"`
FeedLink string `json:"feed_link"`
Icon *[]byte `json:"icon,omitempty"`
HasIcon bool `json:"has_icon"`
}
type CreateFeedParams struct {
Title string
Description string
Link string
FeedLink string
FolderID *int64
}
func (s *Storage) CreateFeed(params CreateFeedParams) *Feed {
title := params.Title
if title == "" {
title = params.FeedLink
}
row := s.db.QueryRow(`
insert into feeds (title, description, link, feed_link, folder_id)
values (:title, :description, :link, :feed_link, :folder_id)
on conflict (feed_link) do update set folder_id = :folder_id
returning id`,
sql.Named("title", title),
sql.Named("description", params.Description),
sql.Named("link", params.Link),
sql.Named("feed_link", params.FeedLink),
sql.Named("folder_id", params.FolderID),
)
var id int64
err := row.Scan(&id)
if err != nil {
log.Print(err)
return nil
}
return &Feed{
Id: id,
Title: title,
Description: params.Description,
Link: params.Link,
FeedLink: params.FeedLink,
FolderId: params.FolderID,
}
}
func (s *Storage) DeleteFeed(feedId int64) bool {
result, err := s.db.Exec(`delete from feeds where id = :id`, sql.Named("id", feedId))
if err != nil {
log.Print(err)
return false
}
nrows, err := result.RowsAffected()
if err != nil {
if err != sql.ErrNoRows {
log.Print(err)
}
return false
}
return nrows == 1
}
type UpdateFeedParams struct {
Title *string
FeedLink *string
FolderID Nullable[int64]
Icon Nullable[[]byte]
}
func (s *Storage) UpdateFeed(feedId int64, params UpdateFeedParams) (bool, error) {
_, err := s.db.Exec(`
update feeds set
title = coalesce(:title, title),
feed_link = coalesce(:feed_link, feed_link),
folder_id = case when :update_folder_id then :folder_id else folder_id end,
icon = case when :update_icon then :icon else icon end
where id = :id
`,
sql.Named("id", feedId),
sql.Named("title", params.Title),
sql.Named("feed_link", params.FeedLink),
sql.Named("update_folder_id", params.FolderID.Set),
sql.Named("folder_id", params.FolderID.Value),
sql.Named("update_icon", params.Icon.Set),
sql.Named("icon", params.Icon.Value),
)
if err != nil {
log.Print(err)
return false, err
}
return true, nil
}
func (s *Storage) ListFeeds() []Feed {
result := make([]Feed, 0)
rows, err := s.db.Query(`
select id, folder_id, title, description, link, feed_link,
ifnull(length(icon), 0) > 0 as has_icon
from feeds
order by title collate nocase
`)
if err != nil {
log.Print(err)
return result
}
for rows.Next() {
var f Feed
err = rows.Scan(
&f.Id,
&f.FolderId,
&f.Title,
&f.Description,
&f.Link,
&f.FeedLink,
&f.HasIcon,
)
if err != nil {
log.Print(err)
return result
}
result = append(result, f)
}
return result
}
func (s *Storage) GetFeed(id int64) *Feed {
var f Feed
err := s.db.QueryRow(`
select
id, folder_id, title, link, feed_link,
icon, ifnull(icon, '') != '' as has_icon
from feeds where id = :id
`, sql.Named("id", id)).Scan(
&f.Id, &f.FolderId, &f.Title, &f.Link, &f.FeedLink,
&f.Icon, &f.HasIcon,
)
if err != nil {
if err != sql.ErrNoRows {
log.Print(err)
}
return nil
}
return &f
}

View File

@@ -0,0 +1,90 @@
package storage
import (
"reflect"
"testing"
)
func TestCreateFeed(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
if feed1 == nil || feed1.Id == 0 {
t.Fatal("expected feed")
}
feed2 := db.GetFeed(feed1.Id)
if feed2 == nil || !reflect.DeepEqual(feed1, feed2) {
t.Fatal("invalid feed")
}
}
func TestCreateFeedSameLink(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", FeedLink: "http://example1.com/feed.xml"})
if feed1 == nil || feed1.Id == 0 {
t.Fatal("expected feed")
}
for range 10 {
db.CreateFeed(CreateFeedParams{Title: "title", FeedLink: "http://example2.com/feed.xml"})
}
feed2 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example1.com/feed.xml"})
if feed1.Id != feed2.Id {
t.Fatalf("expected the same feed.\nwant: %#v\nhave: %#v", feed1, feed2)
}
}
func TestReadFeed(t *testing.T) {
db := testDB()
if db.GetFeed(100500) != nil {
t.Fatal("cannot get nonexistent feed")
}
feed1 := db.CreateFeed(CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
feed2 := db.CreateFeed(CreateFeedParams{Title: "feed 2", Link: "http://example2.com", FeedLink: "http://example2.com/feed.xml"})
feeds := db.ListFeeds()
if !reflect.DeepEqual(feeds, []Feed{*feed1, *feed2}) {
t.Fatalf("invalid feed list: %#v", feeds)
}
}
func TestUpdateFeed(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "feed 1", Link: "http://example1.com", FeedLink: "http://example1.com/feed.xml"})
folder := db.CreateFolder("test")
icon := []byte("icon")
title := "newtitle"
db.UpdateFeed(feed1.Id, UpdateFeedParams{
Title: &title,
FolderID: SetNullable(&folder.Id),
Icon: SetNullable(&icon),
})
feed2 := db.GetFeed(feed1.Id)
if feed2.Title != "newtitle" {
t.Error("invalid title")
}
if feed2.FolderId == nil || *feed2.FolderId != folder.Id {
t.Error("invalid folder")
}
if !feed2.HasIcon || string(*feed2.Icon) != "icon" {
t.Error("invalid icon")
}
}
func TestDeleteFeed(t *testing.T) {
db := testDB()
feed1 := db.CreateFeed(CreateFeedParams{Title: "title", Link: "http://example.com", FeedLink: "http://example.com/feed.xml"})
if db.DeleteFeed(100500) {
t.Error("cannot delete what does not exist")
}
if !db.DeleteFeed(feed1.Id) {
t.Fatal("did not delete existing feed")
}
if db.GetFeed(feed1.Id) != nil {
t.Fatal("feed still exists")
}
}

View File

@@ -0,0 +1,119 @@
package storage
import (
"database/sql"
"time"
)
type FeedState struct {
FeedID int64
LastRefreshed time.Time
LastError string
HTTPLastModified string
HTTPEtag string
}
func (s *Storage) ListFeedStates() ([]FeedState, error) {
rows, err := s.db.Query(`
select
feed_id
, last_refreshed
, last_error
, http_lmod
, http_etag
from feed_states
`)
if err != nil {
return nil, err
}
defer rows.Close()
states := make([]FeedState, 0)
for rows.Next() {
var state FeedState
err := rows.Scan(
&state.FeedID,
&state.LastRefreshed,
&state.LastError,
&state.HTTPLastModified,
&state.HTTPEtag,
)
if err != nil {
return nil, err
}
states = append(states, state)
}
return states, nil
}
func (s *Storage) GetFeedState(feedID int64) (*FeedState, error) {
var state FeedState
err := s.db.QueryRow(`
select
feed_id
, last_refreshed
, last_error
, http_lmod
, http_etag
from feed_states where feed_id = :id
`, sql.Named("id", feedID)).Scan(
&state.FeedID,
&state.LastRefreshed,
&state.LastError,
&state.HTTPLastModified,
&state.HTTPEtag,
)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
return &state, nil
}
type UpdateFeedStateParams struct {
LastRefreshed *time.Time
LastError *string
HTTPLastModified *string
HTTPEtag *string
}
func (s *Storage) UpdateFeedState(feedID int64, params UpdateFeedStateParams) (bool, error) {
lastError := params.LastError
if lastError != nil && *lastError == "" {
lastError = nil
}
_, err := s.db.Exec(`
insert into feed_states (
feed_id
, last_refreshed
, last_error
, http_lmod
, http_etag
)
values (
:id
, coalesce(:last_refreshed, 0)
, coalesce(:last_error, '')
, coalesce(:http_lmod, '')
, coalesce(:http_etag, '')
)
on conflict (feed_id) do update set
last_refreshed = coalesce(:last_refreshed, last_refreshed),
last_error = coalesce(:last_error, last_error),
http_lmod = coalesce(:http_lmod, http_lmod),
http_etag = coalesce(:http_etag, http_etag)
`,
sql.Named("id", feedID),
sql.Named("last_refreshed", params.LastRefreshed),
sql.Named("last_error", params.LastError),
sql.Named("http_lmod", params.HTTPLastModified),
sql.Named("http_etag", params.HTTPEtag),
)
if err != nil {
return false, err
}
return true, nil
}

View File

@@ -0,0 +1,129 @@
package storage
import (
"testing"
"time"
)
func TestUpdateFeedState_Full(t *testing.T) {
s := testDB()
defer s.Close()
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
now := time.Now().UTC().Truncate(time.Second)
errMsg := "error"
lmod := "today"
etag := "v1"
ok, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
LastRefreshed: &now,
LastError: &errMsg,
HTTPLastModified: &lmod,
HTTPEtag: &etag,
})
if err != nil {
t.Fatal(err)
}
if !ok {
t.Error("expected true")
}
state, err := s.GetFeedState(f.Id)
if err != nil {
t.Fatal(err)
}
if state == nil {
t.Fatal("expected state, got nil")
}
if !state.LastRefreshed.Equal(now) {
t.Errorf("expected %v, got %v", now, state.LastRefreshed)
}
if state.LastError != errMsg {
t.Errorf("expected %s, got %v", errMsg, state.LastError)
}
if state.HTTPLastModified != lmod {
t.Errorf("expected %s, got %s", lmod, state.HTTPLastModified)
}
if state.HTTPEtag != etag {
t.Errorf("expected %s, got %s", etag, state.HTTPEtag)
}
}
func TestUpdateFeedState_Partial(t *testing.T) {
s := testDB()
defer s.Close()
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
etag := "v1"
s.UpdateFeedState(f.Id, UpdateFeedStateParams{HTTPEtag: &etag})
newErr := "new error"
_, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
LastError: &newErr,
})
if err != nil {
t.Fatal(err)
}
state, err := s.GetFeedState(f.Id)
if err != nil {
t.Fatal(err)
}
if state.LastError != newErr {
t.Errorf("expected %s, got %v", newErr, state.LastError)
}
if state.HTTPEtag != etag {
t.Errorf("etag should be unchanged, got %s", state.HTTPEtag)
}
}
func TestUpdateFeedState_ClearError(t *testing.T) {
s := testDB()
defer s.Close()
f := s.CreateFeed(CreateFeedParams{Title: "Test", FeedLink: "http://example.com"})
errMsg := "error"
s.UpdateFeedState(f.Id, UpdateFeedStateParams{LastError: &errMsg})
empty := ""
_, err := s.UpdateFeedState(f.Id, UpdateFeedStateParams{
LastError: &empty,
})
if err != nil {
t.Fatal(err)
}
state, err := s.GetFeedState(f.Id)
if err != nil {
t.Fatal(err)
}
if state.LastError != "" {
t.Errorf("expected empty error string, got %v", state.LastError)
}
}
func TestListFeedStates(t *testing.T) {
s := testDB()
defer s.Close()
f1 := s.CreateFeed(CreateFeedParams{Title: "F1", FeedLink: "L1"})
f2 := s.CreateFeed(CreateFeedParams{Title: "F2", FeedLink: "L2"})
errMsg := "fail"
s.UpdateFeedState(f1.Id, UpdateFeedStateParams{LastError: &errMsg})
s.UpdateFeedState(f2.Id, UpdateFeedStateParams{HTTPEtag: ptr("e")})
states, err := s.ListFeedStates()
if err != nil {
t.Fatal(err)
}
if len(states) != 2 {
t.Errorf("expected 2 states, got %d", len(states))
}
}
func ptr[T any](v T) *T {
return &v
}

View File

@@ -0,0 +1,85 @@
package storage
import (
"database/sql"
"log"
)
type Folder struct {
Id int64 `json:"id"`
Title string `json:"title"`
IsExpanded bool `json:"is_expanded"`
}
func (s *Storage) CreateFolder(title string) *Folder {
expanded := true
row := s.db.QueryRow(`
insert into folders (title, is_expanded) values (:title, :is_expanded)
on conflict (title) do update set title = :title
returning id`,
sql.Named("title", title),
sql.Named("is_expanded", expanded),
)
var id int64
err := row.Scan(&id)
if err != nil {
log.Print(err)
return nil
}
return &Folder{Id: id, Title: title, IsExpanded: expanded}
}
func (s *Storage) DeleteFolder(folderId int64) bool {
_, err := s.db.Exec(`delete from folders where id = :id`, sql.Named("id", folderId))
if err != nil {
log.Print(err)
}
return err == nil
}
type UpdateFolderParams struct {
Title *string
IsExpanded *bool
}
func (s *Storage) UpdateFolder(folderId int64, params UpdateFolderParams) (bool, error) {
_, err := s.db.Exec(`
update folders set
title = coalesce(:title, title),
is_expanded = coalesce(:is_expanded, is_expanded)
where id = :id
`,
sql.Named("id", folderId),
sql.Named("title", params.Title),
sql.Named("is_expanded", params.IsExpanded),
)
if err != nil {
log.Print(err)
return false, err
}
return true, nil
}
func (s *Storage) ListFolders() []Folder {
result := make([]Folder, 0)
rows, err := s.db.Query(`
select id, title, is_expanded
from folders
order by title collate nocase
`)
if err != nil {
log.Print(err)
return result
}
for rows.Next() {
var f Folder
err = rows.Scan(&f.Id, &f.Title, &f.IsExpanded)
if err != nil {
log.Print(err)
return result
}
result = append(result, f)
}
return result
}

View File

@@ -0,0 +1,78 @@
package storage
import (
"testing"
)
func TestUpdateFolder(t *testing.T) {
db := testDB()
folder := db.CreateFolder("old title")
if folder.IsExpanded != true {
t.Fatal("expected folder to be expanded by default")
}
t.Run("rename only", func(t *testing.T) {
newTitle := "new title"
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
Title: &newTitle,
})
if !ok || err != nil {
t.Fatalf("UpdateFolder failed: %v", err)
}
folders := db.ListFolders()
if len(folders) != 1 || folders[0].Title != "new title" {
t.Errorf("expected title to be updated, got %s", folders[0].Title)
}
if folders[0].IsExpanded != true {
t.Error("expected expansion state to remain unchanged")
}
})
t.Run("toggle expanded only", func(t *testing.T) {
isExpanded := false
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
IsExpanded: &isExpanded,
})
if !ok || err != nil {
t.Fatalf("UpdateFolder failed: %v", err)
}
folders := db.ListFolders()
if len(folders) != 1 || folders[0].IsExpanded != false {
t.Errorf("expected is_expanded to be false, got %v", folders[0].IsExpanded)
}
if folders[0].Title != "new title" {
t.Error("expected title to remain unchanged")
}
})
t.Run("update both", func(t *testing.T) {
bothTitle := "both"
isExpanded := true
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{
Title: &bothTitle,
IsExpanded: &isExpanded,
})
if !ok || err != nil {
t.Fatalf("UpdateFolder failed: %v", err)
}
folders := db.ListFolders()
if len(folders) != 1 || folders[0].Title != "both" || folders[0].IsExpanded != true {
t.Errorf("expected both to be updated, got title=%s expanded=%v", folders[0].Title, folders[0].IsExpanded)
}
})
t.Run("update none", func(t *testing.T) {
ok, err := db.UpdateFolder(folder.Id, UpdateFolderParams{})
if !ok || err != nil {
t.Fatalf("UpdateFolder failed: %v", err)
}
folders := db.ListFolders()
if len(folders) != 1 || folders[0].Title != "both" || folders[0].IsExpanded != true {
t.Errorf("expected no changes, got title=%s expanded=%v", folders[0].Title, folders[0].IsExpanded)
}
})
}

424
src/storage/sqlite/item.go Normal file
View File

@@ -0,0 +1,424 @@
package storage
import (
"database/sql"
"database/sql/driver"
"encoding/json"
"fmt"
"log"
"sort"
"strings"
"time"
)
type ItemStatus int
const (
UNREAD ItemStatus = 0
READ ItemStatus = 1
STARRED ItemStatus = 2
)
var StatusRepresentations = map[ItemStatus]string{
UNREAD: "unread",
READ: "read",
STARRED: "starred",
}
var StatusValues = map[string]ItemStatus{
"unread": UNREAD,
"read": READ,
"starred": STARRED,
}
func (s ItemStatus) MarshalJSON() ([]byte, error) {
return json.Marshal(StatusRepresentations[s])
}
func (s *ItemStatus) UnmarshalJSON(b []byte) error {
var str string
if err := json.Unmarshal(b, &str); err != nil {
return err
}
*s = StatusValues[str]
return nil
}
type MediaLink struct {
URL string `json:"url"`
Type string `json:"type"`
Description string `json:"description,omitempty"`
}
type MediaLinks []MediaLink
func (m *MediaLinks) Scan(src any) error {
switch data := src.(type) {
case []byte:
return json.Unmarshal(data, m)
case string:
return json.Unmarshal([]byte(data), m)
default:
return nil
}
}
func (m MediaLinks) Value() (driver.Value, error) {
return json.Marshal(m)
}
type Item struct {
Id int64 `json:"id"`
GUID string `json:"guid"`
FeedId int64 `json:"feed_id"`
Title string `json:"title"`
Link string `json:"link"`
Content string `json:"content,omitempty"`
Date time.Time `json:"date"`
Status ItemStatus `json:"status"`
MediaLinks MediaLinks `json:"media_links"`
}
type ItemFilter struct {
FolderID *int64
FeedID *int64
Status *ItemStatus
Search *string
After *int64
IDs *[]int64
SinceID *int64
MaxID *int64
Before *time.Time
}
type MarkFilter struct {
FolderID *int64
FeedID *int64
Before *time.Time
}
type ItemList []Item
func (list ItemList) Len() int {
return len(list)
}
func (list ItemList) SortKey(i int) string {
return list[i].Date.Format(time.RFC3339) + "::" + list[i].GUID
}
func (list ItemList) Less(i, j int) bool {
return list.SortKey(i) < list.SortKey(j)
}
func (list ItemList) Swap(i, j int) {
list[i], list[j] = list[j], list[i]
}
func (s *Storage) CreateItems(items []Item) bool {
tx, err := s.db.Begin()
if err != nil {
log.Print(err)
return false
}
now := time.Now().UTC()
itemsSorted := ItemList(items)
sort.Sort(itemsSorted)
for _, item := range itemsSorted {
_, err = tx.Exec(`
insert into items (
guid, feed_id, title, link, date,
content, media_links,
date_arrived, last_arrived, status
)
values (
:guid, :feed_id, :title, :link, strftime('%Y-%m-%d %H:%M:%f', :date),
:content, :media_links,
:date_arrived, :last_arrived, :status
)
on conflict (feed_id, guid) do update set
last_arrived = :last_arrived`,
sql.Named("guid", item.GUID),
sql.Named("feed_id", item.FeedId),
sql.Named("title", item.Title),
sql.Named("link", item.Link),
sql.Named("date", item.Date),
sql.Named("content", item.Content),
sql.Named("media_links", item.MediaLinks),
sql.Named("date_arrived", now),
sql.Named("last_arrived", now),
sql.Named("status", UNREAD),
)
if err != nil {
log.Print(err)
if err = tx.Rollback(); err != nil {
log.Print(err)
return false
}
return false
}
}
if err = tx.Commit(); err != nil {
log.Print(err)
return false
}
return true
}
func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []any) {
cond := make([]string, 0)
args := make([]any, 0)
if filter.FolderID != nil {
cond = append(cond, "i.feed_id in (select id from feeds where folder_id = :folder_id)")
args = append(args, sql.Named("folder_id", *filter.FolderID))
}
if filter.FeedID != nil {
cond = append(cond, "i.feed_id = :feed_id")
args = append(args, sql.Named("feed_id", *filter.FeedID))
}
if filter.Status != nil {
cond = append(cond, "i.status = :status")
args = append(args, sql.Named("status", *filter.Status))
}
if filter.Search != nil {
words := strings.Fields(*filter.Search)
terms := make([]string, len(words))
for idx, word := range words {
terms[idx] = word + "*"
}
cond = append(
cond,
"i.id in (select rowid as id from search where search match :search)",
)
args = append(args, sql.Named("search", strings.Join(terms, " ")))
}
if filter.After != nil {
compare := ">"
if newestFirst {
compare = "<"
}
cond = append(
cond,
fmt.Sprintf(
"(i.date, i.id) %s (select date, id from items where id = :after_id)",
compare,
),
)
args = append(args, sql.Named("after_id", *filter.After))
}
if filter.IDs != nil && len(*filter.IDs) > 0 {
qmarks := make([]string, len(*filter.IDs))
for i, id := range *filter.IDs {
name := fmt.Sprintf("id%d", i)
qmarks[i] = ":" + name
args = append(args, sql.Named(name, id))
}
cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")")
}
if filter.SinceID != nil {
cond = append(cond, "i.id > :since_id")
args = append(args, sql.Named("since_id", filter.SinceID))
}
if filter.MaxID != nil {
cond = append(cond, "i.id < :max_id")
args = append(args, sql.Named("max_id", filter.MaxID))
}
if filter.Before != nil {
cond = append(cond, "i.date < :before")
args = append(args, sql.Named("before", filter.Before))
}
predicate := "1"
if len(cond) > 0 {
predicate = strings.Join(cond, " and ")
}
return predicate, args
}
func (s *Storage) CountItems() int {
var count int
err := s.db.QueryRow(`select count(*) from items`).Scan(&count)
if err != nil {
log.Print(err)
return 0
}
return count
}
func (s *Storage) ListItems(
filter ItemFilter,
limit int,
newestFirst bool,
withContent bool,
) []Item {
predicate, args := listQueryPredicate(filter, newestFirst)
result := make([]Item, 0)
order := "date desc, id desc"
if !newestFirst {
order = "date asc, id asc"
}
if filter.IDs != nil || filter.SinceID != nil {
order = "i.id asc"
}
if filter.MaxID != nil {
order = "i.id desc"
}
selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.media_links"
if withContent {
selectCols += ", i.content"
} else {
selectCols += ", '' as content"
}
query := fmt.Sprintf(`
select %s
from items i
where %s
order by %s
limit %d
`, selectCols, predicate, order, limit)
rows, err := s.db.Query(query, args...)
if err != nil {
log.Print(err)
return result
}
for rows.Next() {
var x Item
err = rows.Scan(
&x.Id, &x.GUID, &x.FeedId,
&x.Title, &x.Link, &x.Date,
&x.Status, &x.MediaLinks, &x.Content,
)
if err != nil {
log.Print(err)
return result
}
result = append(result, x)
}
return result
}
func (s *Storage) GetItem(id int64) *Item {
i := &Item{}
err := s.db.QueryRow(`
select
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
i.date, i.status, i.media_links
from items i
where i.id = :id
`, sql.Named("id", id)).Scan(
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
&i.Date, &i.Status, &i.MediaLinks,
)
if err != nil {
log.Print(err)
return nil
}
return i
}
func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
_, err := s.db.Exec(`update items set status = :status where id = :id`,
sql.Named("status", status),
sql.Named("id", item_id),
)
return err == nil
}
func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
predicate, args := listQueryPredicate(ItemFilter{
FolderID: filter.FolderID,
FeedID: filter.FeedID,
Before: filter.Before,
}, false)
query := fmt.Sprintf(`
update items as i set status = %d
where %s and i.status != %d
`, READ, predicate, STARRED)
_, err := s.db.Exec(query, args...)
if err != nil {
log.Print(err)
}
return err == nil
}
type FeedStat struct {
FeedId int64 `json:"feed_id"`
UnreadCount int64 `json:"unread"`
StarredCount int64 `json:"starred"`
}
func (s *Storage) FeedStats() []FeedStat {
result := make([]FeedStat, 0)
rows, err := s.db.Query(fmt.Sprintf(`
select
feed_id,
sum(case status when %d then 1 else 0 end),
sum(case status when %d then 1 else 0 end)
from items
group by feed_id
`, UNREAD, STARRED))
if err != nil {
log.Print(err)
return result
}
for rows.Next() {
stat := FeedStat{}
rows.Scan(&stat.FeedId, &stat.UnreadCount, &stat.StarredCount)
result = append(result, stat)
}
return result
}
var (
itemsKeepSize = 50
itemsKeepDays = 90
)
// Delete old articles from the database to cleanup space.
//
// The rules:
// - Never delete starred entries.
// - Keep at least 50 latest items for each feed.
// - Delete entries older than 90 days relative to the latest arrived item in the same feed.
func (s *Storage) DeleteOldItems() {
result, err := s.db.Exec(`
delete from items
where id in (
select id
from (
select
id,
row_number() over (partition by feed_id order by date desc) as rn,
last_arrived,
max(last_arrived) over (partition by feed_id) as max_la
from items
where status != :starred_status
)
where rn > :keep_size
and last_arrived < datetime(max_la, :keep_days_limit)
)`,
sql.Named("starred_status", STARRED),
sql.Named("keep_size", itemsKeepSize),
sql.Named("keep_days_limit", fmt.Sprintf("-%d days", itemsKeepDays)),
)
if err != nil {
log.Print(err)
return
}
numDeleted, err := result.RowsAffected()
if err == nil && numDeleted > 0 {
log.Printf("Deleted %d old items", numDeleted)
if _, err := s.db.Exec("vacuum"); err != nil {
log.Print(err)
}
}
}

View File

@@ -0,0 +1,515 @@
package storage
import (
"database/sql"
"log"
"reflect"
"strconv"
"testing"
"testing/synctest"
"time"
)
/*
- folder1
- feed11
- item111 (unread)
- item112 (read)
- item113 (starred)
- feed12
- item121 (unread)
- item122 (read)
- folder2
- feed21
- item211 (read)
- item212 (starred)
- feed01
- item011 (unread)
- item012 (read)
- item013 (starred)
*/
type testItemScope struct {
feed11, feed12 *Feed
feed21, feed01 *Feed
folder1, folder2 *Folder
}
func testItemsSetup(db *Storage) testItemScope {
folder1 := db.CreateFolder("folder1")
folder2 := db.CreateFolder("folder2")
feed11 := db.CreateFeed(CreateFeedParams{Title: "feed11", FeedLink: "http://test.com/feed11.xml", FolderID: &folder1.Id})
feed12 := db.CreateFeed(CreateFeedParams{Title: "feed12", FeedLink: "http://test.com/feed12.xml", FolderID: &folder1.Id})
feed21 := db.CreateFeed(CreateFeedParams{Title: "feed21", FeedLink: "http://test.com/feed21.xml", FolderID: &folder2.Id})
feed01 := db.CreateFeed(CreateFeedParams{Title: "feed01", FeedLink: "http://test.com/feed01.xml"})
now := time.Now()
db.CreateItems([]Item{
// feed11
{GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)},
{
GUID: "item112",
FeedId: feed11.Id,
Title: "title112",
Date: now.Add(time.Hour * 24 * 2),
}, // read
{
GUID: "item113",
FeedId: feed11.Id,
Title: "title113",
Date: now.Add(time.Hour * 24 * 3),
}, // starred
// feed12
{GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)},
{
GUID: "item122",
FeedId: feed12.Id,
Title: "title122",
Date: now.Add(time.Hour * 24 * 5),
}, // read
// feed21
{
GUID: "item211",
FeedId: feed21.Id,
Title: "title211",
Date: now.Add(time.Hour * 24 * 6),
}, // read
{
GUID: "item212",
FeedId: feed21.Id,
Title: "title212",
Date: now.Add(time.Hour * 24 * 7),
}, // starred
// feed01
{GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)},
{
GUID: "item012",
FeedId: feed01.Id,
Title: "title012",
Date: now.Add(time.Hour * 24 * 9),
}, // read
{
GUID: "item013",
FeedId: feed01.Id,
Title: "title013",
Date: now.Add(time.Hour * 24 * 10),
}, // starred
})
db.db.Exec(
`update items set status = :status where guid in ("item112", "item122", "item211", "item012")`,
sql.Named("status", READ),
)
db.db.Exec(
`update items set status = :status where guid in ("item113", "item212", "item013")`,
sql.Named("status", STARRED),
)
return testItemScope{
feed11: feed11,
feed12: feed12,
feed21: feed21,
feed01: feed01,
folder1: folder1,
folder2: folder2,
}
}
func getItem(db *Storage, guid string) *Item {
i := &Item{}
err := db.db.QueryRow(`
select
i.id, i.guid, i.feed_id, i.title, i.link, i.content,
i.date, i.status, i.media_links
from items i
where i.guid = :guid
`, sql.Named("guid", guid)).Scan(
&i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content,
&i.Date, &i.Status, &i.MediaLinks,
)
if err != nil {
log.Fatal(err)
}
return i
}
func getItemGuids(items []Item) []string {
guids := make([]string, 0)
for _, item := range items {
guids = append(guids, item.GUID)
}
return guids
}
func TestListItems(t *testing.T) {
db := testDB()
scope := testItemsSetup(db)
// filter by folder_id
have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
want := []string{"item111", "item112", "item113", "item121", "item122"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
want = []string{"item211", "item212"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
// filter by feed_id
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
want = []string{"item111", "item112", "item113"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
want = []string{"item011", "item012", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
// filter by status
var starred ItemStatus = STARRED
have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
want = []string{"item113", "item212", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
var unread ItemStatus = UNREAD
have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
want = []string{"item111", "item121", "item011"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
// limit
have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
want = []string{"item111", "item112"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
// filter by search
search1 := "title111"
have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
want = []string{"item111"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
// sort by date
have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
want = []string{"item013", "item012", "item011", "item212"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
}
func TestListItemsPaginated(t *testing.T) {
db := testDB()
testItemsSetup(db)
item012 := getItem(db, "item012")
item121 := getItem(db, "item121")
// all, newest first
have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
want := []string{"item011", "item212", "item211"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
// unread, newest first
unread := UNREAD
have = getItemGuids(
db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false),
)
want = []string{"item011", "item121", "item111"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
// starred, oldest first
starred := STARRED
have = getItemGuids(
db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false),
)
want = []string{"item212", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
}
func TestMarkItemsRead(t *testing.T) {
// NOTE: starred items must not be marked as read
var read ItemStatus = READ
db1 := testDB()
testItemsSetup(db1)
db1.MarkItemsRead(MarkFilter{})
have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
want := []string{
"item111", "item112", "item121", "item122",
"item211", "item011", "item012",
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
db2 := testDB()
scope2 := testItemsSetup(db2)
db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
want = []string{
"item111", "item112", "item121", "item122",
"item211", "item012",
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
db3 := testDB()
scope3 := testItemsSetup(db3)
db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
want = []string{
"item111", "item112", "item122",
"item211", "item012",
}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
t.Logf("have: %#v", have)
t.Fail()
}
}
func TestDeleteOldItems(t *testing.T) {
now := time.Now().UTC()
starred := STARRED
t.Run("keeps at least 50 items", func(t *testing.T) {
db := testDB()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]Item, 100)
for i := range 100 {
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Hour * 24)}
}
db.CreateItems(items)
// // Set 1 recent (latest), 100 old (100 days ago)
db.db.Exec(`update items set last_arrived = :la where guid = "99"`, sql.Named("la", now))
db.db.Exec(`update items set last_arrived = :la where guid != "99"`, sql.Named("la", now.Add(-time.Hour*24*100)))
db.DeleteOldItems()
var have int
db.db.QueryRow("select count(*) from items where feed_id = ?", feed.Id).Scan(&have)
if have != 50 {
t.Errorf("expected 50 items, have %d", have)
}
})
t.Run("keeps all less than 90 days old", func(t *testing.T) {
db := testDB()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]Item, 100)
for i := 0; i < 100; i++ {
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
}
db.CreateItems(items)
// Latest item at "now"
// All others at 80 days ago (keep)
db.db.Exec(`update items set last_arrived = :la where guid = "99"`, sql.Named("la", now))
db.db.Exec(`update items set last_arrived = :la where guid != "99"`, sql.Named("la", now.Add(-time.Hour*24*80)))
db.DeleteOldItems()
var have int
db.db.QueryRow("select count(*) from items where feed_id = ?", feed.Id).Scan(&have)
if have != 100 {
t.Errorf("expected 100 items, have %d", have)
}
})
t.Run("keeps starred", func(t *testing.T) {
db := testDB()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
items := make([]Item, 100)
for i := 0; i < 100; i++ {
items[i] = Item{GUID: strconv.Itoa(i), FeedId: feed.Id, Date: now.Add(time.Duration(i) * time.Second)}
}
db.CreateItems(items)
// Set all to 100 days ago, except one recent
db.db.Exec(`update items set last_arrived = :la`, sql.Named("la", now.Add(-time.Hour*24*100)))
db.db.Exec(`update items set last_arrived = :la where guid = "99"`, sql.Named("la", now))
// Star 10 old items that would otherwise be deleted (rn > 50 and old)
db.db.Exec(`update items set status = :s where cast(guid as integer) < 10`, sql.Named("s", starred))
db.DeleteOldItems()
var have int
db.db.QueryRow("select count(*) from items where feed_id = ?", feed.Id).Scan(&have)
// 50 (limit) + 10 (starred) = 60 items should remain.
if have != 60 {
t.Errorf("expected 60 items, have %d", have)
}
})
}
func TestCreateItemsLastArrived(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
db := testDB()
defer db.db.Close()
feed := db.CreateFeed(CreateFeedParams{Title: "test feed", FeedLink: "http://example.com/feed"})
item := Item{
GUID: "item1",
FeedId: feed.Id,
Title: "Title 1",
Date: time.Now(),
}
// 1. Initial creation
db.CreateItems([]Item{item})
var lastArrived1 time.Time
err := db.db.QueryRow("select last_arrived from items where guid = ?", item.GUID).Scan(&lastArrived1)
if err != nil {
t.Fatal(err)
}
time.Sleep(time.Second * 10)
// 2. Update on conflict
db.CreateItems([]Item{item})
var lastArrived2 time.Time
err = db.db.QueryRow("select last_arrived from items where guid = ?", item.GUID).Scan(&lastArrived2)
if err != nil {
t.Fatal(err)
}
if !lastArrived2.After(lastArrived1) {
t.Errorf("expected last_arrived to be updated. old: %v, new: %v", lastArrived1, lastArrived2)
}
})
}
func TestSearch(t *testing.T) {
db := testDB()
defer db.Close()
feed := db.CreateFeed(CreateFeedParams{Title: "f", FeedLink: "http://f.xml"})
db.CreateItems([]Item{
{
GUID: "i1",
FeedId: feed.Id,
Title: "Hello World",
Content: "This is a <b>test</b> of the <i>emergency</i> broadcast system.",
},
{
GUID: "i2",
FeedId: feed.Id,
Title: "FTS5 Unicode",
Content: "Unicode support with characters like: Привет, 世界, 🚀",
},
{
GUID: "i3",
FeedId: feed.Id,
Title: "Hidden Tag",
Content: `<div class="secret-class">Don't find me by my class name</div>`,
},
})
// 1. Basic search
s1 := "emergency"
have := getItemGuids(db.ListItems(ItemFilter{Search: &s1}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("basic search failed: expected [i1], got %v", have)
}
// 2. HTML stripping: Should find text, but NOT the tags
s2 := "test"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s2}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("html text search failed: expected [i1], got %v", have)
}
s3 := "secret-class"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s3}, 10, true, false))
if len(have) > 0 {
t.Errorf("html tag search should have failed but found: %v", have)
}
// 3. Multi-word (AND)
s4 := "broadcast system"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s4}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("multi-word search failed: expected [i1], got %v", have)
}
// 4. Unicode
s5 := "Привет"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s5}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i2"}) {
t.Errorf("unicode search failed: expected [i2], got %v", have)
}
s6 := "世界"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s6}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i2"}) {
t.Errorf("unicode search (CJK) failed: expected [i2], got %v", have)
}
// 5. Trigger: Update
db.db.Exec("update items set title = 'Updated Title' where guid = 'i1'")
s7 := "Updated"
have = getItemGuids(db.ListItems(ItemFilter{Search: &s7}, 10, true, false))
if !reflect.DeepEqual(have, []string{"i1"}) {
t.Errorf("update trigger failed: expected [i1], got %v", have)
}
// 6. Trigger: Delete
db.db.Exec("delete from items where guid = 'i1'")
have = getItemGuids(db.ListItems(ItemFilter{Search: &s7}, 10, true, false))
if len(have) > 0 {
t.Errorf("delete trigger failed: found deleted item: %v", have)
}
}

View File

@@ -0,0 +1,422 @@
package storage
import (
"database/sql"
"fmt"
"log"
"time"
)
var migrations = []func(*sql.Tx) error{
m01_initial,
m02_feed_states_and_errors,
m03_on_delete_actions,
m04_item_podcasturl,
m05_move_description_to_content,
m06_fill_missing_dates,
m07_add_feed_size,
m08_normalize_datetime,
m09_change_item_index,
m10_add_item_medialinks,
m11_add_item_last_arrived,
m12_remove_feed_sizes,
m13_consolidate_feed_states,
m14_upgrade_fts5,
}
var maxVersion = int64(len(migrations))
func migrate(db *sql.DB) error {
var version int64
if err := db.QueryRow("pragma user_version").Scan(&version); err != nil {
return err
}
if version >= maxVersion {
return nil
}
log.Printf("db version is %d. migrating to %d", version, maxVersion)
for v := version + 1; v <= maxVersion; v++ {
// Migrations altering schema using a sequence of steps due to SQLite limitations.
// Must come with `pragma foreign_key_check` at the end. See:
// "Making Other Kinds Of Table Schema Changes"
// https://www.sqlite.org/lang_altertable.html
trickyAlteration := (v == 3)
log.Printf("[migration:%d] starting", v)
if trickyAlteration {
db.Exec("pragma foreign_keys=off;")
}
err := migrateVersion(v, db)
if trickyAlteration {
db.Exec("pragma foreign_keys=on;")
}
if err != nil {
return err
}
log.Printf("[migration:%d] done", v)
}
return nil
}
func migrateVersion(v int64, db *sql.DB) error {
var err error
var tx *sql.Tx
migratefunc := migrations[v-1]
if tx, err = db.Begin(); err != nil {
log.Printf("[migration:%d] failed to start transaction", v)
return err
}
if err = migratefunc(tx); err != nil {
log.Printf("[migration:%d] failed to migrate", v)
tx.Rollback()
return err
}
if _, err = tx.Exec(fmt.Sprintf("pragma user_version = %d", v)); err != nil {
log.Printf("[migration:%d] failed to bump version", v)
tx.Rollback()
return err
}
if err = tx.Commit(); err != nil {
log.Printf("[migration:%d] failed to commit changes", v)
return err
}
return nil
}
func m01_initial(tx *sql.Tx) error {
sql := `
create table if not exists folders (
id integer primary key autoincrement,
title text not null,
is_expanded boolean not null default false
);
create unique index if not exists idx_folder_title on folders(title);
create table if not exists feeds (
id integer primary key autoincrement,
folder_id references folders(id),
title text not null,
description text,
link text,
feed_link text not null,
icon blob
);
create index if not exists idx_feed_folder_id on feeds(folder_id);
create unique index if not exists idx_feed_feed_link on feeds(feed_link);
create table if not exists items (
id integer primary key autoincrement,
guid string not null,
feed_id references feeds(id),
title text,
link text,
description text,
content text,
author text,
date datetime,
date_updated datetime,
date_arrived datetime,
status integer,
image text,
search_rowid integer
);
create index if not exists idx_item_feed_id on items(feed_id);
create index if not exists idx_item_status on items(status);
create index if not exists idx_item_search_rowid on items(search_rowid);
create unique index if not exists idx_item_guid on items(feed_id, guid);
create table if not exists settings (
key string primary key,
val blob
);
create virtual table if not exists search using fts4(title, description, content);
create trigger if not exists del_item_search after delete on items begin
delete from search where rowid = old.search_rowid;
end;
`
_, err := tx.Exec(sql)
return err
}
func m02_feed_states_and_errors(tx *sql.Tx) error {
sql := `
create table if not exists http_states (
feed_id references feeds(id) unique,
last_refreshed datetime not null,
-- http header fields --
last_modified string not null,
etag string not null
);
create table if not exists feed_errors (
feed_id references feeds(id) unique,
error string
);
`
_, err := tx.Exec(sql)
return err
}
func m03_on_delete_actions(tx *sql.Tx) error {
sql := `
-- 01. create altered tables
create table if not exists new_feeds (
id integer primary key autoincrement,
folder_id references folders(id) on delete set null,
title text not null,
description text,
link text,
feed_link text not null,
icon blob
);
create table if not exists new_items (
id integer primary key autoincrement,
guid string not null,
feed_id references feeds(id) on delete cascade,
title text,
link text,
description text,
content text,
author text,
date datetime,
date_updated datetime,
date_arrived datetime,
status integer,
image text,
search_rowid integer
);
create table if not exists new_http_states (
feed_id references feeds(id) on delete cascade unique,
last_refreshed datetime not null,
last_modified string not null,
etag string not null
);
create table if not exists new_feed_errors (
feed_id references feeds(id) on delete cascade unique,
error string
);
-- 02. transfer data into new tables
insert into new_feeds select * from feeds;
insert into new_items select * from items;
insert into new_http_states select * from http_states;
insert into new_feed_errors select * from feed_errors;
-- 03. drop old tables
drop table feeds;
drop table items;
drop table http_states;
drop table feed_errors;
-- 04. rename new tables
alter table new_feeds rename to feeds;
alter table new_items rename to items;
alter table new_http_states rename to http_states;
alter table new_feed_errors rename to feed_errors;
-- 05. reconstruct indexes & triggers
create index if not exists idx_feed_folder_id on feeds(folder_id);
create unique index if not exists idx_feed_feed_link on feeds(feed_link);
create index if not exists idx_item_feed_id on items(feed_id);
create index if not exists idx_item_status on items(status);
create index if not exists idx_item_search_rowid on items(search_rowid);
create unique index if not exists idx_item_guid on items(feed_id, guid);
create trigger if not exists del_item_search after delete on items begin
delete from search where rowid = old.search_rowid;
end;
-- 06. check consistency
pragma foreign_key_check;
`
_, err := tx.Exec(sql)
return err
}
func m04_item_podcasturl(tx *sql.Tx) error {
sql := `
alter table items add column podcast_url text
`
_, err := tx.Exec(sql)
return err
}
func m05_move_description_to_content(tx *sql.Tx) error {
sql := `
update items set content=description
where length(content) = 0 or content is null
`
_, err := tx.Exec(sql)
return err
}
func m06_fill_missing_dates(tx *sql.Tx) error {
sql := `
update items set date = 0 where date is null;
`
_, err := tx.Exec(sql)
return err
}
func m07_add_feed_size(tx *sql.Tx) error {
sql := `
create table if not exists feed_sizes (
feed_id references feeds(id) on delete cascade unique,
size integer not null default 0
);
`
_, err := tx.Exec(sql)
return err
}
func m08_normalize_datetime(tx *sql.Tx) error {
rows, err := tx.Query(`select id, date_arrived from items;`)
if err != nil {
return err
}
for rows.Next() {
var id int64
var dateArrived time.Time
err = rows.Scan(&id, &dateArrived)
if err != nil {
return err
}
_, err = tx.Exec(`update items set date_arrived = :date_arrived where id = :id;`,
sql.Named("date_arrived", dateArrived.UTC()),
sql.Named("id", id),
)
if err != nil {
return err
}
}
_, err = tx.Exec(`update items set date = strftime('%Y-%m-%d %H:%M:%f', date);`)
return err
}
func m09_change_item_index(tx *sql.Tx) error {
sql := `
drop index if exists idx_item_status;
create index if not exists idx_item__date_id_status on items(date,id,status);
`
_, err := tx.Exec(sql)
return err
}
func m10_add_item_medialinks(tx *sql.Tx) error {
sql := `
alter table items add column media_links json;
update items set media_links = case
when coalesce(image, '') != '' and coalesce(podcast_url, '') != ''
then json_array(json_object('type', 'image', 'url', image), json_object('type', 'audio', 'url', podcast_url))
when coalesce(image, '') != ''
then json_array(json_object('type', 'image', 'url', image))
when coalesce(podcast_url, '') != ''
then json_array(json_object('type', 'audio', 'url', podcast_url))
else null
end;
alter table items drop column image;
alter table items drop column podcast_url;
`
_, err := tx.Exec(sql)
return err
}
func m11_add_item_last_arrived(tx *sql.Tx) error {
sql := `alter table items add column last_arrived datetime`
_, err := tx.Exec(sql)
return err
}
func m12_remove_feed_sizes(tx *sql.Tx) error {
_, err := tx.Exec(`drop table if exists feed_sizes`)
return err
}
func m13_consolidate_feed_states(tx *sql.Tx) error {
sql := `
create table feed_states (
feed_id references feeds(id) on delete cascade unique
, last_refreshed datetime not null default 0
, last_error string not null default ''
, http_lmod string not null default ''
, http_etag string not null default ''
);
insert into feed_states (
feed_id
, last_refreshed
, last_error
, http_lmod
, http_etag
)
select
f.id
, coalesce(h.last_refreshed, 0)
, coalesce(e.error, '')
, coalesce(h.last_modified, '')
, coalesce(h.etag, '')
from feeds f
left join http_states h on f.id = h.feed_id
left join feed_errors e on f.id = e.feed_id
where h.feed_id is not null or e.feed_id is not null;
drop table http_states;
drop table feed_errors;
`
_, err := tx.Exec(sql)
return err
}
func m14_upgrade_fts5(tx *sql.Tx) error {
sql := `
-- 1. Drop old FTS4 table and trigger
drop table if exists search;
drop trigger if exists del_item_search;
-- 2. Remove search_rowid from items
drop index if exists idx_item_search_rowid;
alter table items drop column search_rowid;
-- 3. Create FTS5 virtual table
create virtual table search using fts5(
title, content,
content='items',
content_rowid='id',
tokenize='unicode61'
);
-- 4. Create triggers for automatic FTS sync
create trigger items_ai after insert on items begin
insert into search(rowid, title, content) values (new.id, new.title, strip_html(new.content));
end;
create trigger items_ad after delete on items begin
insert into search(search, rowid, title, content) values('delete', old.id, old.title, strip_html(old.content));
end;
create trigger items_au after update on items begin
insert into search(search, rowid, title, content) values('delete', old.id, old.title, strip_html(old.content));
insert into search(rowid, title, content) values (new.id, new.title, strip_html(new.content));
end;
-- 5. Populate FTS5 table with existing data
insert into search(rowid, title, content) select id, title, strip_html(content) from items;
`
_, err := tx.Exec(sql)
return err
}

View File

@@ -0,0 +1,171 @@
package storage
import (
"database/sql"
"encoding/json"
"log"
)
type Settings struct {
Filter string `json:"filter"`
Feed string `json:"feed"`
FeedListWidth int `json:"feed_list_width"`
ItemListWidth int `json:"item_list_width"`
SortNewestFirst bool `json:"sort_newest_first"`
ThemeName string `json:"theme_name"`
ThemeFont string `json:"theme_font"`
ThemeSize int `json:"theme_size"`
RefreshRate int64 `json:"refresh_rate"`
Language string `json:"language"`
}
func (s Settings) Map() map[string]any {
return map[string]any{
"filter": s.Filter,
"feed": s.Feed,
"feed_list_width": s.FeedListWidth,
"item_list_width": s.ItemListWidth,
"sort_newest_first": s.SortNewestFirst,
"theme_name": s.ThemeName,
"theme_font": s.ThemeFont,
"theme_size": s.ThemeSize,
"refresh_rate": s.RefreshRate,
"language": s.Language,
}
}
func settingsDefaults() Settings {
return Settings{
Filter: "",
Feed: "",
FeedListWidth: 300,
ItemListWidth: 300,
SortNewestFirst: true,
ThemeName: "light",
ThemeFont: "",
ThemeSize: 1,
RefreshRate: 0,
Language: "en",
}
}
func (s *Storage) GetSettings() Settings {
result := settingsDefaults()
rows, err := s.db.Query(`select key, val from settings;`)
if err != nil {
log.Print(err)
return result
}
defer rows.Close()
for rows.Next() {
var key string
var val []byte
rows.Scan(&key, &val)
switch key {
case "filter":
json.Unmarshal(val, &result.Filter)
case "feed":
json.Unmarshal(val, &result.Feed)
case "feed_list_width":
json.Unmarshal(val, &result.FeedListWidth)
case "item_list_width":
json.Unmarshal(val, &result.ItemListWidth)
case "sort_newest_first":
json.Unmarshal(val, &result.SortNewestFirst)
case "theme_name":
json.Unmarshal(val, &result.ThemeName)
case "theme_font":
json.Unmarshal(val, &result.ThemeFont)
case "theme_size":
json.Unmarshal(val, &result.ThemeSize)
case "refresh_rate":
json.Unmarshal(val, &result.RefreshRate)
case "language":
json.Unmarshal(val, &result.Language)
}
}
return result
}
type UpdateSettingsParams struct {
Filter *string `json:"filter"`
Feed *string `json:"feed"`
FeedListWidth *int `json:"feed_list_width"`
ItemListWidth *int `json:"item_list_width"`
SortNewestFirst *bool `json:"sort_newest_first"`
ThemeName *string `json:"theme_name"`
ThemeFont *string `json:"theme_font"`
ThemeSize *int `json:"theme_size"`
RefreshRate *int64 `json:"refresh_rate"`
Language *string `json:"language"`
}
func (s *Storage) UpdateSettings(params UpdateSettingsParams) bool {
tx, err := s.db.Begin()
if err != nil {
log.Print(err)
return false
}
defer tx.Rollback()
update := func(key string, val any) error {
valEncoded, err := json.Marshal(val)
if err != nil {
return err
}
_, err = tx.Exec(`
insert into settings (key, val) values (:key, :val)
on conflict (key) do update set val=:val`,
sql.Named("key", key),
sql.Named("val", valEncoded),
)
return err
}
var errs []error
if params.Filter != nil {
errs = append(errs, update("filter", *params.Filter))
}
if params.Feed != nil {
errs = append(errs, update("feed", *params.Feed))
}
if params.FeedListWidth != nil {
errs = append(errs, update("feed_list_width", *params.FeedListWidth))
}
if params.ItemListWidth != nil {
errs = append(errs, update("item_list_width", *params.ItemListWidth))
}
if params.SortNewestFirst != nil {
errs = append(errs, update("sort_newest_first", *params.SortNewestFirst))
}
if params.ThemeName != nil {
errs = append(errs, update("theme_name", *params.ThemeName))
}
if params.ThemeFont != nil {
errs = append(errs, update("theme_font", *params.ThemeFont))
}
if params.ThemeSize != nil {
errs = append(errs, update("theme_size", *params.ThemeSize))
}
if params.RefreshRate != nil {
errs = append(errs, update("refresh_rate", *params.RefreshRate))
}
if params.Language != nil {
errs = append(errs, update("language", *params.Language))
}
for _, err := range errs {
if err != nil {
log.Print(err)
return false
}
}
if err := tx.Commit(); err != nil {
log.Print(err)
return false
}
return true
}

View File

@@ -0,0 +1,150 @@
package storage
import (
"reflect"
"strings"
"testing"
)
func TestSettingsDefaults(t *testing.T) {
s := testDB()
defer s.Close()
settings := s.GetSettings()
defaults := settingsDefaults()
if !reflect.DeepEqual(settings, defaults) {
t.Errorf("expected defaults %+v, got %+v", defaults, settings)
}
}
func TestUpdateSettings(t *testing.T) {
s := testDB()
defer s.Close()
params := UpdateSettingsParams{
ThemeName: ptr("night"),
FeedListWidth: ptr(400),
RefreshRate: ptr(int64(15)),
}
if ok := s.UpdateSettings(params); !ok {
t.Fatal("UpdateSettings failed")
}
settings := s.GetSettings()
if settings.ThemeName != "night" {
t.Errorf("expected theme_name night, got %s", settings.ThemeName)
}
if settings.FeedListWidth != 400 {
t.Errorf("expected feed_list_width 400, got %d", settings.FeedListWidth)
}
if settings.RefreshRate != 15 {
t.Errorf("expected refresh_rate 15, got %d", settings.RefreshRate)
}
}
func TestGetSettings(t *testing.T) {
s := testDB()
defer s.Close()
s.UpdateSettings(UpdateSettingsParams{Language: ptr("fr")})
settings := s.GetSettings()
if settings.Language != "fr" {
t.Errorf("expected fr, got %v", settings.Language)
}
if settings.ThemeName != "light" {
t.Errorf("expected light, got %v", settings.ThemeName)
}
}
func TestSettingsExhaustive(t *testing.T) {
s := testDB()
defer s.Close()
settingsType := reflect.TypeOf(Settings{})
paramsType := reflect.TypeOf(UpdateSettingsParams{})
settings := s.GetSettings()
m := settings.Map()
for i := 0; i < settingsType.NumField(); i++ {
field := settingsType.Field(i)
jsonTag := field.Tag.Get("json")
if jsonTag == "" {
t.Errorf("Field %s missing json tag", field.Name)
continue
}
// json tags might have options like "name,omitempty", take only the first part
jsonKey := strings.Split(jsonTag, ",")[0]
// 1. Check Map()
if _, ok := m[jsonKey]; !ok {
t.Errorf("Key %q (from field %s) missing from Settings.Map()", jsonKey, field.Name)
}
// 2. Check UpdateSettingsParams
foundInParams := false
for j := 0; j < paramsType.NumField(); j++ {
pField := paramsType.Field(j)
pJsonTag := strings.Split(pField.Tag.Get("json"), ",")[0]
if pJsonTag == jsonKey {
foundInParams = true
// Also check it's a pointer
if pField.Type.Kind() != reflect.Ptr {
t.Errorf("Field %s in UpdateSettingsParams should be a pointer", pField.Name)
}
break
}
}
if !foundInParams {
t.Errorf("Key %q (from field %s) missing from UpdateSettingsParams", jsonKey, field.Name)
}
// 3. Test round-trip update
// We'll create a new UpdateSettingsParams and set ONLY this field
paramsValue := reflect.New(paramsType).Elem()
for j := 0; j < paramsType.NumField(); j++ {
pField := paramsType.Field(j)
pJsonTag := strings.Split(pField.Tag.Get("json"), ",")[0]
if pJsonTag == jsonKey {
// Create a new value of the underlying type
val := reflect.New(field.Type).Elem()
switch field.Type.Kind() {
case reflect.String:
val.SetString("test_" + jsonKey)
case reflect.Int, reflect.Int64:
val.SetInt(42)
case reflect.Bool:
val.SetBool(false)
}
paramsValue.Field(j).Set(val.Addr())
break
}
}
if ok := s.UpdateSettings(paramsValue.Interface().(UpdateSettingsParams)); !ok {
t.Errorf("UpdateSettings failed for %q", jsonKey)
}
updated := s.GetSettings()
updatedValue := reflect.ValueOf(updated).Field(i)
switch field.Type.Kind() {
case reflect.String:
if updatedValue.String() != "test_"+jsonKey {
t.Errorf("Round-trip failed for %q: expected %q, got %q (check UpdateSettings/GetSettings switch)", jsonKey, "test_"+jsonKey, updatedValue.String())
}
case reflect.Int, reflect.Int64:
if updatedValue.Int() != 42 {
t.Errorf("Round-trip failed for %q: expected 42, got %d (check UpdateSettings/GetSettings switch)", jsonKey, updatedValue.Int())
}
case reflect.Bool:
if updatedValue.Bool() != false {
t.Errorf("Round-trip failed for %q: expected false, got %v (check UpdateSettings/GetSettings switch)", jsonKey, updatedValue.Bool())
}
}
}
}

View File

@@ -0,0 +1,53 @@
package storage
import (
"database/sql"
"log"
"strings"
"github.com/mattn/go-sqlite3"
"github.com/nkanaev/yarr/src/content/htmlutil"
)
func init() {
sql.Register("sqlite3_yarr", &sqlite3.SQLiteDriver{
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
return conn.RegisterFunc("strip_html", htmlutil.ExtractText, true)
},
})
}
type Storage struct {
db *sql.DB
}
type Nullable[T any] struct {
Set bool
Value *T
}
func SetNullable[T any](v *T) Nullable[T] {
return Nullable[T]{Set: true, Value: v}
}
func New(path string) (*Storage, error) {
if pos := strings.IndexRune(path, '?'); pos == -1 {
params := "_journal=WAL&_sync=NORMAL&_busy_timeout=5000&cache=shared"
log.Printf("opening db with params: %s", params)
path = path + "?" + params
}
db, err := sql.Open("sqlite3_yarr", path)
if err != nil {
return nil, err
}
if err = migrate(db); err != nil {
return nil, err
}
return &Storage{db: db}, nil
}
func (s *Storage) Close() error {
return s.db.Close()
}

View File

@@ -0,0 +1,28 @@
package storage
import (
"io"
"log"
"os"
"testing"
)
func testDB() *Storage {
log.SetOutput(io.Discard)
db, err := New(":memory:")
if err != nil {
panic(err)
}
log.SetOutput(os.Stderr)
return db
}
func TestStorage(t *testing.T) {
db, err := New(":memory:")
if err != nil {
t.Fatal(err)
}
if db == nil {
t.Fatal("no db")
}
}