change http state storage

server/crawler.go

@@ -258,7 +258,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 	var res *http.Response
 	var err error
 
-	httpState := db.GetHTTPState(f.FeedLink)
+	httpState := db.GetHTTPState(f.Id)
 	if httpState != nil {
 		res, err = defaultClient.getConditional(f.FeedLink, httpState.LastModified, httpState.Etag)
 	} else {
@@ -282,7 +282,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 	lastModified := res.Header.Get("Last-Modified")
 	etag := res.Header.Get("Etag")
 	if lastModified != "" || etag != "" {
-		db.SetHTTPState(f.FeedLink, storage.HTTPState{LastModified: lastModified, Etag: etag})
+		db.SetHTTPState(f.Id, lastModified, etag)
 	}
 
 	feedparser := gofeed.NewParser()

storage/http.go

@@ -1,32 +1,46 @@
 package storage
 
+import (
+	"time"
+)
+
 type HTTPState struct {
+	FeedID        int64
+	LastRefreshed time.Time
+
 	LastModified  string
 	Etag          string
 }
 
-func (s *Storage) GetHTTPState(url string) *HTTPState {
+func (s *Storage) GetHTTPState(feedID int64) *HTTPState {
 	row := s.db.QueryRow(`
-		select last_modified, etag
+		select feed_id, last_refreshed, last_modified, etag
-		from http_state where url = ?
+		from http_states where feed_id = ?
-	`, url)
+	`, feedID)
 
 	if row == nil {
 		return nil
 	}
 
 	var state HTTPState
-	row.Scan(&state.LastModified, &state.Etag)
+	row.Scan(
+		&state.FeedID,
+		&state.LastRefreshed,
+		&state.LastModified,
+		&state.Etag,
+	)
 	return &state
 }
 
-func (s *Storage) SetHTTPState(url string, state HTTPState) {
+func (s *Storage) SetHTTPState(feedID int64, lastModified, etag string) {
 	_, err := s.db.Exec(`
-		insert into http_state (url, last_modified, etag)
+		insert into http_states (feed_id, last_modified, etag, last_refreshed)
-		values (?, ?, ?)
+		values (?, ?, ?, datetime())
-		on conflict (url) do update set last_modified = ?, etag = ?`,
+		on conflict (feed_id) do update set last_modified = ?, etag = ?, last_refreshed = datetime()`,
-		url, state.LastModified, state.Etag,
+		// insert
-		state.LastModified, state.Etag,
+		feedID, lastModified, etag,
+		// upsert
+		lastModified, etag,
 	)
 	if err != nil {
 		s.log.Print(err)

storage/storage.go

@@ -62,8 +62,11 @@ create trigger if not exists del_item_search after delete on items begin
   delete from search where rowid = old.search_rowid;
 end;
 
-create table if not exists http_state (
+create table if not exists http_states (
- url            string not null primary key,
+ feed_id        references feeds(id) unique,
+ last_refreshed datetime not null,
+
+ -- http header fields --
  last_modified  string not null,
  etag           string not null
 );