diff --git a/src/storage/feed.go b/src/storage/feed.go index 2033de6..a415955 100644 --- a/src/storage/feed.go +++ b/src/storage/feed.go @@ -194,3 +194,15 @@ func (s *Storage) GetFeedErrors() map[int64]string { } return errors } + +func (s *Storage) SetFeedSize(feedId int64, size int) { + _, err := s.db.Exec(` + insert into feed_sizes (feed_id, size) + values (?, ?) + on conflict (feed_id) do update set size = excluded.size`, + feedId, size, + ) + if err != nil { + log.Print(err) + } +} diff --git a/src/storage/item.go b/src/storage/item.go index 59227a2..9441d47 100644 --- a/src/storage/item.go +++ b/src/storage/item.go @@ -292,6 +292,13 @@ func (s *Storage) SyncSearch() { } } + +// TODO: better naming +var ( + itemsKeepSize = 100 + itemsKeepDays = 90 +) + func (s *Storage) DeleteOldItems() { rows, err := s.db.Query(fmt.Sprintf(` select feed_id, count(*) as num_items @@ -318,7 +325,7 @@ func (s *Storage) DeleteOldItems() { delete from items where feed_id = ? and status != ? and date_arrived < ?`, feedId, STARRED, - time.Now().Add(-time.Hour*24*90), // 90 days + time.Now().Add(-time.Hour*time.Duration(24*itemsKeepDays)), ) if err != nil { log.Print(err) diff --git a/src/storage/item_test.go b/src/storage/item_test.go index 89c209b..6e8c1cf 100644 --- a/src/storage/item_test.go +++ b/src/storage/item_test.go @@ -3,6 +3,7 @@ package storage import ( "log" "reflect" + "strconv" "testing" "time" ) @@ -45,14 +46,14 @@ func testItemsSetup(db *Storage) testItemScope { db.CreateItems([]Item{ // feed11 {GUID: "item111", FeedId: feed11.Id, Title: "title111", Date: now.Add(time.Hour * 24 * 1)}, - {GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)}, // read - {GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)}, // starred + {GUID: "item112", FeedId: feed11.Id, Title: "title112", Date: now.Add(time.Hour * 24 * 2)}, // read + {GUID: "item113", FeedId: feed11.Id, Title: "title113", Date: now.Add(time.Hour * 24 * 3)}, // starred // feed12 {GUID: "item121", FeedId: feed12.Id, Title: "title121", Date: now.Add(time.Hour * 24 * 4)}, - {GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)}, // read + {GUID: "item122", FeedId: feed12.Id, Title: "title122", Date: now.Add(time.Hour * 24 * 5)}, // read // feed21 - {GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)}, // read - {GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)}, // starred + {GUID: "item211", FeedId: feed21.Id, Title: "title211", Date: now.Add(time.Hour * 24 * 6)}, // read + {GUID: "item212", FeedId: feed21.Id, Title: "title212", Date: now.Add(time.Hour * 24 * 7)}, // starred // feed01 {GUID: "item011", FeedId: feed01.Id, Title: "title011", Date: now.Add(time.Hour * 24 * 8)}, {GUID: "item012", FeedId: feed01.Id, Title: "title012", Date: now.Add(time.Hour * 24 * 9)}, // read @@ -271,3 +272,57 @@ func TestMarkItemsRead(t *testing.T) { t.Fail() } } + +func TestDeleteOldItems(t *testing.T) { + extraItems := 10 + + now := time.Now() + db := testDB() + feed := db.CreateFeed("feed", "", "", "http://test.com/feed11.xml", nil) + + items := make([]Item, 0) + for i := 0; i < itemsKeepSize+extraItems; i++ { + istr := strconv.Itoa(i) + items = append(items, Item{ + GUID: istr, + FeedId: feed.Id, + Title: istr, + Date: now.Add(time.Hour * time.Duration(i)), + }) + } + db.CreateItems(items) + + db.SetFeedSize(feed.Id, len(items)) + var feedSize int + err := db.db.QueryRow( + `select size from feed_sizes where feed_id = ?`, feed.Id, + ).Scan(&feedSize) + if err != nil { + t.Fatal(err) + } + if feedSize != itemsKeepSize+extraItems { + t.Fatalf( + "expected feed size to get updated\nwant: %d\nhave: %d", + itemsKeepSize+extraItems, + feedSize, + ) + } + + _, err = db.db.Exec( + `update items set date_arrived = ?`, + now.Add(-time.Hour*time.Duration(itemsKeepDays*24)), + ) + if err != nil { + t.Fatal(err) + } + + db.DeleteOldItems() + feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false) + if len(feedItems) != itemsKeepSize { + t.Fatalf( + "invalid number of old items kept\nwant: %d\nhave: %d", + itemsKeepSize, + len(feedItems), + ) + } +} diff --git a/src/storage/migration.go b/src/storage/migration.go index a4688bd..1fb4642 100644 --- a/src/storage/migration.go +++ b/src/storage/migration.go @@ -13,6 +13,7 @@ var migrations = []func(*sql.Tx) error{ m04_item_podcasturl, m05_move_description_to_content, m06_fill_missing_dates, + m07_add_feed_size, } var maxVersion = int64(len(migrations)) @@ -259,3 +260,14 @@ func m06_fill_missing_dates(tx *sql.Tx) error { _, err := tx.Exec(sql) return err } + +func m07_add_feed_size(tx *sql.Tx) error { + sql := ` + create table if not exists feed_sizes ( + feed_id references feeds(id) unique, + size integer not null default 0 + ); + ` + _, err := tx.Exec(sql) + return err +} diff --git a/src/storage/storage_test.go b/src/storage/storage_test.go index 47dc800..09908c1 100644 --- a/src/storage/storage_test.go +++ b/src/storage/storage_test.go @@ -11,6 +11,7 @@ func testDB() *Storage { log.SetOutput(io.Discard) db, _ := New(":memory:") log.SetOutput(os.Stderr) + log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) return db }