diff --git a/server/crawler.go b/server/crawler.go index e52d6c8..c0c12ef 100644 --- a/server/crawler.go +++ b/server/crawler.go @@ -258,7 +258,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) { var res *http.Response var err error - httpState := db.GetHTTPState(f.FeedLink) + httpState := db.GetHTTPState(f.Id) if httpState != nil { res, err = defaultClient.getConditional(f.FeedLink, httpState.LastModified, httpState.Etag) } else { @@ -282,7 +282,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) { lastModified := res.Header.Get("Last-Modified") etag := res.Header.Get("Etag") if lastModified != "" || etag != "" { - db.SetHTTPState(f.FeedLink, storage.HTTPState{LastModified: lastModified, Etag: etag}) + db.SetHTTPState(f.Id, lastModified, etag) } feedparser := gofeed.NewParser() diff --git a/storage/http.go b/storage/http.go index 09a589b..2a13915 100644 --- a/storage/http.go +++ b/storage/http.go @@ -1,32 +1,46 @@ package storage +import ( + "time" +) + type HTTPState struct { - LastModified string - Etag string + FeedID int64 + LastRefreshed time.Time + + LastModified string + Etag string } -func (s *Storage) GetHTTPState(url string) *HTTPState { +func (s *Storage) GetHTTPState(feedID int64) *HTTPState { row := s.db.QueryRow(` - select last_modified, etag - from http_state where url = ? - `, url) + select feed_id, last_refreshed, last_modified, etag + from http_states where feed_id = ? + `, feedID) if row == nil { return nil } var state HTTPState - row.Scan(&state.LastModified, &state.Etag) + row.Scan( + &state.FeedID, + &state.LastRefreshed, + &state.LastModified, + &state.Etag, + ) return &state } -func (s *Storage) SetHTTPState(url string, state HTTPState) { +func (s *Storage) SetHTTPState(feedID int64, lastModified, etag string) { _, err := s.db.Exec(` - insert into http_state (url, last_modified, etag) - values (?, ?, ?) - on conflict (url) do update set last_modified = ?, etag = ?`, - url, state.LastModified, state.Etag, - state.LastModified, state.Etag, + insert into http_states (feed_id, last_modified, etag, last_refreshed) + values (?, ?, ?, datetime()) + on conflict (feed_id) do update set last_modified = ?, etag = ?, last_refreshed = datetime()`, + // insert + feedID, lastModified, etag, + // upsert + lastModified, etag, ) if err != nil { s.log.Print(err) diff --git a/storage/storage.go b/storage/storage.go index 86b5831..3e98696 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -62,8 +62,11 @@ create trigger if not exists del_item_search after delete on items begin delete from search where rowid = old.search_rowid; end; -create table if not exists http_state ( - url string not null primary key, +create table if not exists http_states ( + feed_id references feeds(id) unique, + last_refreshed datetime not null, + + -- http header fields -- last_modified string not null, etag string not null );