delete old items based on feed size

src/storage/item.go

@@ -61,7 +61,7 @@ type ItemFilter struct {
 	FeedID   *int64
 	Status   *ItemStatus
 	Search   *string
-	After  *int64
+	After    *int64
 }
 
 type MarkFilter struct {
@@ -292,52 +292,70 @@ func (s *Storage) SyncSearch() {
 	}
 }
 
-
-// TODO: better naming
 var (
 	itemsKeepSize = 100
 	itemsKeepDays = 90
 )
 
+// Delete old articles from the database to cleanup space.
+//
+// The rules:
+// * Never delete starred entries
+// * Take each feed capacity (number of entries provided by the feed)
+//   into account (see `SetFeedSize`, default: 100).
+//   This prevents old items from reappearing after the cleanup.
+// * Keep entries for a certain period (default: 90 days).
 func (s *Storage) DeleteOldItems() {
-	rows, err := s.db.Query(fmt.Sprintf(`
+	rows, err := s.db.Query(`
-		select feed_id, count(*) as num_items
+		select
-		from items
+			i.feed_id,
-		where status != %d
+			max(coalesce(s.size, 0), ?) as max_items,
-		group by feed_id
+			count(*) as num_items
-		having num_items > 50
+		from items i
-	`, STARRED))
+		left outer join feed_sizes s on s.feed_id = i.feed_id
+		where status != ?
+		group by i.feed_id
+	`, itemsKeepSize, STARRED)
 
 	if err != nil {
 		log.Print(err)
 		return
 	}
 
-	feedIds := make([]int64, 0)
+	feedLimits := make(map[int64]int64, 0)
 	for rows.Next() {
-		var id int64
+		var feedId, limit int64
-		rows.Scan(&id, nil)
+		rows.Scan(&feedId, &limit, nil)
-		feedIds = append(feedIds, id)
+		feedLimits[feedId] = limit
 	}
 
-	for _, feedId := range feedIds {
+	for feedId, limit := range feedLimits {
 		result, err := s.db.Exec(`
-			delete from items where feed_id = ? and status != ? and date_arrived < ?`,
+			delete from items
+			where id in (
+				select i.id
+				from items i
+				where i.feed_id = ? and status != ?
+				order by date desc
+				limit -1 offset ?
+			) and date_arrived < ?
+			`,
 			feedId,
 			STARRED,
+			limit,
 			time.Now().Add(-time.Hour*time.Duration(24*itemsKeepDays)),
 		)
 		if err != nil {
 			log.Print(err)
 			return
 		}
-		num, err := result.RowsAffected()
+		numDeleted, err := result.RowsAffected()
 		if err != nil {
 			log.Print(err)
 			return
 		}
-		if num > 0 {
+		if numDeleted > 0 {
-			log.Printf("Deleted %d old items (%d)", num, feedId)
+			log.Printf("Deleted %d old items (feed: %d)", numDeleted, feedId)
 		}
 	}
 }

src/storage/item_test.go

@@ -292,7 +292,7 @@ func TestDeleteOldItems(t *testing.T) {
 	}
 	db.CreateItems(items)
 	
-	db.SetFeedSize(feed.Id, len(items))
+	db.SetFeedSize(feed.Id, itemsKeepSize)
 	var feedSize int
 	err := db.db.QueryRow(
 		`select size from feed_sizes where feed_id = ?`, feed.Id,
@@ -300,7 +300,7 @@ func TestDeleteOldItems(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if feedSize != itemsKeepSize+extraItems {
+	if feedSize != itemsKeepSize {
 		t.Fatalf(
 			"expected feed size to get updated\nwant: %d\nhave: %d", 
 			itemsKeepSize+extraItems,