From 6acf9af8870138083822013224a47acc6ccd7da9 Mon Sep 17 00:00:00 2001 From: Nazar Kanaev Date: Mon, 5 Apr 2021 21:29:52 +0100 Subject: [PATCH] remove item.date_updated, item.description & item.author fields in storage --- src/server/routes.go | 1 - src/storage/item.go | 80 +++++++++++++++++----------------------- src/storage/migration.go | 3 ++ src/worker/crawler.go | 28 +++++++------- 4 files changed, 51 insertions(+), 61 deletions(-) diff --git a/src/server/routes.go b/src/server/routes.go index 0821c8b..0798739 100644 --- a/src/server/routes.go +++ b/src/server/routes.go @@ -248,7 +248,6 @@ func (s *Server) handleItem(c *router.Context) { return } item.Content = sanitizer.Sanitize(item.Link, item.Content) - item.Description = sanitizer.Sanitize(item.Link, item.Description) c.JSON(http.StatusOK, item) } else if c.Req.Method == "PUT" { diff --git a/src/storage/item.go b/src/storage/item.go index c7ee80b..0948d13 100644 --- a/src/storage/item.go +++ b/src/storage/item.go @@ -44,19 +44,16 @@ func (s *ItemStatus) UnmarshalJSON(b []byte) error { } type Item struct { - Id int64 `json:"id"` - GUID string `json:"guid"` - FeedId int64 `json:"feed_id"` - Title string `json:"title"` - Link string `json:"link"` - Description string `json:"description,omitempty"` - Content string `json:"content,omitempty"` - Author string `json:"author"` - Date *time.Time `json:"date"` - DateUpdated *time.Time `json:"date_updated"` - Status ItemStatus `json:"status"` - Image string `json:"image"` - PodcastURL *string `json:"podcast_url"` + Id int64 `json:"id"` + GUID string `json:"guid"` + FeedId int64 `json:"feed_id"` + Title string `json:"title"` + Link string `json:"link"` + Content string `json:"content,omitempty"` + Date time.Time `json:"date"` + Status ItemStatus `json:"status"` + ImageURL *string `json:"image"` + AudioURL *string `json:"podcast_url"` } type ItemFilter struct { @@ -77,25 +74,21 @@ func (s *Storage) CreateItems(items []Item) bool { log.Print(err) return false } + now := time.Now() for _, item := range items { _, err = tx.Exec(` insert into items ( - guid, feed_id, title, link, description, - content, author, - date, date_updated, date_arrived, - status, image, podcast_url + guid, feed_id, title, link, date, + content, image, podcast_url, + date_arrived, status ) - values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - on conflict (feed_id, guid) do update set - date_updated = ?, date_arrived = ?`, - item.GUID, item.FeedId, item.Title, item.Link, item.Description, - item.Content, item.Author, - item.Date, item.DateUpdated, now, - UNREAD, item.Image, item.PodcastURL, - // upsert values - item.DateUpdated, now, + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict (feed_id, guid) do nothing`, + item.GUID, item.FeedId, item.Title, item.Link, item.Date, + item.Content, item.ImageURL, item.AudioURL, + now, UNREAD, ) if err != nil { log.Print(err) @@ -158,8 +151,9 @@ func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bo query := fmt.Sprintf(` select - i.id, i.guid, i.feed_id, i.title, i.link, - i.author, i.date, i.date_updated, i.status, i.image, i.podcast_url + i.id, i.guid, i.feed_id, + i.title, i.link, i.date, + i.status, i.image, i.podcast_url from items i join feeds f on f.id = i.feed_id where %s @@ -174,17 +168,9 @@ func (s *Storage) ListItems(filter ItemFilter, offset, limit int, newestFirst bo for rows.Next() { var x Item err = rows.Scan( - &x.Id, - &x.GUID, - &x.FeedId, - &x.Title, - &x.Link, - &x.Author, - &x.Date, - &x.DateUpdated, - &x.Status, - &x.Image, - &x.PodcastURL, + &x.Id, &x.GUID, &x.FeedId, + &x.Title, &x.Link, &x.Date, + &x.Status, &x.ImageURL, &x.AudioURL, ) if err != nil { log.Print(err) @@ -199,13 +185,13 @@ func (s *Storage) GetItem(id int64) *Item { i := &Item{} err := s.db.QueryRow(` select - i.id, i.guid, i.feed_id, i.title, i.link, i.content, i.description, - i.author, i.date, i.date_updated, i.status, i.image, i.podcast_url + i.id, i.guid, i.feed_id, i.title, i.link, i.content, + i.date, i.status, i.image, i.podcast_url from items i where i.id = ? `, id).Scan( - &i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content, &i.Description, - &i.Author, &i.Date, &i.DateUpdated, &i.Status, &i.Image, &i.PodcastURL, + &i.Id, &i.GUID, &i.FeedId, &i.Title, &i.Link, &i.Content, + &i.Date, &i.Status, &i.ImageURL, &i.AudioURL, ) if err != nil { log.Print(err) @@ -296,7 +282,7 @@ func (s *Storage) FeedStats() []FeedStat { func (s *Storage) SyncSearch() { rows, err := s.db.Query(` - select id, title, content, description + select id, title, content from items where search_rowid is null; `) @@ -308,14 +294,14 @@ func (s *Storage) SyncSearch() { items := make([]Item, 0) for rows.Next() { var item Item - rows.Scan(&item.Id, &item.Title, &item.Content, &item.Description) + rows.Scan(&item.Id, &item.Title, &item.Content) items = append(items, item) } for _, item := range items { result, err := s.db.Exec(` - insert into search (title, description, content) values (?, ?, ?)`, - item.Title, htmlutil.ExtractText(item.Description), htmlutil.ExtractText(item.Content), + insert into search (title, description, content) values (?, "", ?)`, + item.Title, htmlutil.ExtractText(item.Content), ) if err != nil { log.Print(err) diff --git a/src/storage/migration.go b/src/storage/migration.go index 7795e3c..9b2996f 100644 --- a/src/storage/migration.go +++ b/src/storage/migration.go @@ -240,3 +240,6 @@ func m04_item_podcasturl(tx *sql.Tx) error { _, err := tx.Exec(sql) return err } + + +// TODO: description -> content diff --git a/src/worker/crawler.go b/src/worker/crawler.go index 80c4872..6753984 100644 --- a/src/worker/crawler.go +++ b/src/worker/crawler.go @@ -136,22 +136,24 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item { result := make([]storage.Item, len(items)) for i, item := range items { item := item - var podcastURL *string = nil + var audioURL *string = nil if item.AudioURL != "" { - podcastURL = &item.AudioURL + audioURL = &item.AudioURL + } + var imageURL *string = nil + if item.ImageURL != "" { + imageURL = &item.ImageURL } result[i] = storage.Item{ - GUID: item.GUID, - FeedId: feed.Id, - Title: item.Title, - Link: item.URL, - Description: "", - Content: item.Content, - Author: "", - Date: &item.Date, - Status: storage.UNREAD, - Image: item.ImageURL, - PodcastURL: podcastURL, + GUID: item.GUID, + FeedId: feed.Id, + Title: item.Title, + Link: item.URL, + Content: item.Content, + Date: item.Date, + Status: storage.UNREAD, + ImageURL: imageURL, + AudioURL: audioURL, } } return result