populate search table

build.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-CGO_ENABLED=1 go build -tags sqlite_foreign_keys
+CGO_ENABLED=1 go build -tags "sqlite_foreign_keys sqlite_fts5"

go.mod

@@ -6,4 +6,5 @@ require (
 	github.com/PuerkitoBio/goquery v1.5.1
 	github.com/mattn/go-sqlite3 v1.14.0
 	github.com/mmcdole/gofeed v1.0.0
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e
 )

server/server.go

@@ -37,7 +37,8 @@ func (h *Handler) startJobs() {
 			h.queueSize += val
 		}			
 	}()
-	h.fetchAllFeeds()
+	go h.db.SyncSearch()
+	//h.fetchAllFeeds()
 }
 
 func (h *Handler) fetchFeed(feed storage.Feed) {

storage/item.go

@@ -5,6 +5,7 @@ import (
 	"time"
 	"strings"
 	"encoding/json"
+	"golang.org/x/net/html"
 )
 
 type ItemStatus int
@@ -241,3 +242,60 @@ func (s *Storage) FeedStats() []FeedStat {
 	}
 	return result
 }
+
+func HTMLText(s string) string {
+    tokenizer := html.NewTokenizer(strings.NewReader(s))
+	contents := make([]string, 0)
+    for {
+		token := tokenizer.Next()
+		if token == html.ErrorToken {
+			break
+		}
+		if token == html.TextToken {
+            content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text())))
+            if len(content) > 0 {
+				contents = append(contents, content)
+            }
+		}
+    }
+	return strings.Join(contents, " ")
+}
+
+func (s *Storage) SyncSearch() {
+	rows, err := s.db.Query(`
+		select id, title, content, description
+		from items
+		where search_rowid is null;
+	`)
+	if err != nil {
+		s.log.Print(err)
+		return
+	}
+
+	items := make([]Item, 0)
+	for rows.Next() {
+		var item Item
+		rows.Scan(&item.Id, &item.Title, &item.Content, &item.Description)
+		fmt.Println(item)
+		items = append(items, item)
+	}
+
+	for _, item := range items {
+		result, err := s.db.Exec(`
+			insert into search (title, description, content) values (?, ?, ?)`,
+			item.Title, HTMLText(item.Description), HTMLText(item.Content),
+		)
+		if err != nil {
+			s.log.Print(err)
+			return
+		}
+		if numrows, err := result.RowsAffected(); err == nil && numrows == 1 {
+			if rowId, err := result.LastInsertId(); err == nil {
+				s.db.Exec(
+					`update items set search_rowid = ? where id = ?`,
+					rowId, item.Id,
+				)
+			}
+		}
+	}
+}