package storage import ( "encoding/json" "fmt" "html" "log" "strings" "time" xhtml "golang.org/x/net/html" ) type ItemStatus int const ( UNREAD ItemStatus = 0 READ ItemStatus = 1 STARRED ItemStatus = 2 ) var StatusRepresentations = map[ItemStatus]string{ UNREAD: "unread", READ: "read", STARRED: "starred", } var StatusValues = map[string]ItemStatus{ "unread": UNREAD, "read": READ, "starred": STARRED, } func (s ItemStatus) MarshalJSON() ([]byte, error) { return json.Marshal(StatusRepresentations[s]) } func (s *ItemStatus) UnmarshalJSON(b []byte) error { var str string if err := json.Unmarshal(b, &str); err != nil { return err } *s = StatusValues[str] return nil } type Item struct { Id int64 `json:"id"` GUID string `json:"guid"` FeedId int64 `json:"feed_id"` Title string `json:"title"` Link string `json:"link"` Description string `json:"description"` Content string `json:"content"` Author string `json:"author"` Date *time.Time `json:"date"` DateUpdated *time.Time `json:"date_updated"` Status ItemStatus `json:"status"` Image string `json:"image"` PodcastURL *string `json:"podcast_url"` } type ItemFilter struct { FolderID *int64 FeedID *int64 Status *ItemStatus Search *string } type MarkFilter struct { FolderID *int64 FeedID *int64 } func (s *Storage) CreateItems(items []Item) bool { tx, err := s.db.Begin() if err != nil { log.Print(err) return false } now := time.Now() for _, item := range items { // WILD: some feeds provide only `item.date_updated` (without `item.date_created`) if item.Date == nil { item.Date = item.DateUpdated } // WILD: `item.guid` is not always present if item.GUID == "" { item.GUID = item.Link } _, err = tx.Exec(` insert into items ( guid, feed_id, title, link, description, content, author, date, date_updated, date_arrived, status, image, podcast_url ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict (feed_id, guid) do update set date_updated = ?, date_arrived = ?`, item.GUID, item.FeedId, html.UnescapeString(item.Title), item.Link, item.Description, item.Content, item.Author, item.Date, item.DateUpdated, now, UNREAD, item.Image, item.PodcastURL, // upsert values item.DateUpdated, now, ) if err != nil { log.Print(err) if err = tx.Rollback(); err != nil { log.Print(err) return false } return false } } if err = tx.Commit(); err != nil { log.Print(err) return false } return true } func listQueryPredicate(filter ItemFilter) (string, []interface{}) { cond := make([]string, 0) args := make([]interface{}, 0) if filter.FolderID != nil { cond = append(cond, "f.folder_id = ?") args = append(args, *filter.FolderID) } if filter.FeedID != nil { cond = append(cond, "i.feed_id = ?") args = append(args, *filter.FeedID) } if filter.Status != nil { cond = append(cond, "i.status = ?") args = append(args, *filter.Status) } if filter.Search != nil { words := strings.Fields(*filter.Search) terms := make([]string, len(words)) for idx, word := range words { terms[idx] = word + "*" } cond = append(cond, "i.search_rowid in (select rowid from search where search match ?)") args = append(args, strings.Join(terms, " ")) } predicate := "1" if len(cond) > 0 { predicate = strings.Join(cond, " and ") } return predicate, args } func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bool) []Item { predicate, args := listQueryPredicate(filter) result := make([]Item, 0, 0) order := "date desc" if !newestFirst { order = "date asc" } query := fmt.Sprintf(` select i.id, i.guid, i.feed_id, i.title, i.link, i.description, i.content, i.author, i.date, i.date_updated, i.status, i.image, i.podcast_url from items i join feeds f on f.id = i.feed_id where %s order by %s limit %d offset %d `, predicate, order, limit, offset) rows, err := s.db.Query(query, args...) if err != nil { log.Print(err) return result } for rows.Next() { var x Item err = rows.Scan( &x.Id, &x.GUID, &x.FeedId, &x.Title, &x.Link, &x.Description, &x.Content, &x.Author, &x.Date, &x.DateUpdated, &x.Status, &x.Image, &x.PodcastURL, ) if err != nil { log.Print(err) return result } result = append(result, x) } return result } func (s *Storage) CountItems(filter ItemFilter) int64 { predicate, args := listQueryPredicate(filter) query := fmt.Sprintf(` select count(i.id) from items i join feeds f on f.id = i.feed_id where %s`, predicate) row := s.db.QueryRow(query, args...) if row != nil { var result int64 row.Scan(&result) return result } return 0 } func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool { _, err := s.db.Exec(`update items set status = ? where id = ?`, status, item_id) return err == nil } func (s *Storage) MarkItemsRead(filter MarkFilter) bool { cond := make([]string, 0) args := make([]interface{}, 0) if filter.FolderID != nil { cond = append(cond, "f.folder_id = ?") args = append(args, *filter.FolderID) } if filter.FeedID != nil { cond = append(cond, "i.feed_id = ?") args = append(args, *filter.FeedID) } predicate := "1" if len(cond) > 0 { predicate = strings.Join(cond, " and ") } query := fmt.Sprintf(` update items set status = %d where id in ( select i.id from items i join feeds f on f.id = i.feed_id where %s and i.status != %d ) `, READ, predicate, STARRED) _, err := s.db.Exec(query, args...) if err != nil { log.Print(err) } return err == nil } type FeedStat struct { FeedId int64 `json:"feed_id"` UnreadCount int64 `json:"unread"` StarredCount int64 `json:"starred"` } func (s *Storage) FeedStats() []FeedStat { result := make([]FeedStat, 0) rows, err := s.db.Query(fmt.Sprintf(` select feed_id, sum(case status when %d then 1 else 0 end), sum(case status when %d then 1 else 0 end) from items group by feed_id `, UNREAD, STARRED)) if err != nil { log.Print(err) return result } for rows.Next() { stat := FeedStat{} rows.Scan(&stat.FeedId, &stat.UnreadCount, &stat.StarredCount) result = append(result, stat) } return result } func HTMLText(s string) string { tokenizer := xhtml.NewTokenizer(strings.NewReader(s)) contents := make([]string, 0) for { token := tokenizer.Next() if token == xhtml.ErrorToken { break } if token == xhtml.TextToken { content := strings.TrimSpace(xhtml.UnescapeString(string(tokenizer.Text()))) if len(content) > 0 { contents = append(contents, content) } } } return strings.Join(contents, " ") } func (s *Storage) SyncSearch() { // TODO: cleaning up once feeds/items are deleted? rows, err := s.db.Query(` select id, title, content, description from items where search_rowid is null; `) if err != nil { log.Print(err) return } items := make([]Item, 0) for rows.Next() { var item Item rows.Scan(&item.Id, &item.Title, &item.Content, &item.Description) items = append(items, item) } for _, item := range items { result, err := s.db.Exec(` insert into search (title, description, content) values (?, ?, ?)`, item.Title, HTMLText(item.Description), HTMLText(item.Content), ) if err != nil { log.Print(err) return } if numrows, err := result.RowsAffected(); err == nil && numrows == 1 { if rowId, err := result.LastInsertId(); err == nil { s.db.Exec( `update items set search_rowid = ? where id = ?`, rowId, item.Id, ) } } } } func (s *Storage) DeleteOldItems() { rows, err := s.db.Query(fmt.Sprintf(` select feed_id, count(*) as num_items from items where status != %d group by feed_id having num_items > 50 `, STARRED)) if err != nil { log.Print(err) return } feedIds := make([]int64, 0) for rows.Next() { var id int64 rows.Scan(&id, nil) feedIds = append(feedIds, id) } for _, feedId := range feedIds { result, err := s.db.Exec(` delete from items where feed_id = ? and status != ? and date_arrived < ?`, feedId, STARRED, time.Now().Add(-time.Hour*24*90), // 90 days ) if err != nil { log.Print(err) return } num, err := result.RowsAffected() if err != nil { log.Print(err) return } if num > 0 { log.Printf("Deleted %d old items (%d)", num, feedId) } } }