diff --git a/build.sh b/build.sh index 7d84adc..5e5c41a 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,3 @@ #!/bin/sh -CGO_ENABLED=1 go build -tags sqlite_foreign_keys +CGO_ENABLED=1 go build -tags "sqlite_foreign_keys sqlite_fts5" diff --git a/go.mod b/go.mod index 1ee2e93..6f728bf 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/PuerkitoBio/goquery v1.5.1 github.com/mattn/go-sqlite3 v1.14.0 github.com/mmcdole/gofeed v1.0.0 + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e ) diff --git a/server/server.go b/server/server.go index e5d3935..d55908a 100644 --- a/server/server.go +++ b/server/server.go @@ -37,7 +37,8 @@ func (h *Handler) startJobs() { h.queueSize += val } }() - h.fetchAllFeeds() + go h.db.SyncSearch() + //h.fetchAllFeeds() } func (h *Handler) fetchFeed(feed storage.Feed) { diff --git a/storage/item.go b/storage/item.go index c6bb24d..78b18c0 100644 --- a/storage/item.go +++ b/storage/item.go @@ -5,6 +5,7 @@ import ( "time" "strings" "encoding/json" + "golang.org/x/net/html" ) type ItemStatus int @@ -241,3 +242,60 @@ func (s *Storage) FeedStats() []FeedStat { } return result } + +func HTMLText(s string) string { + tokenizer := html.NewTokenizer(strings.NewReader(s)) + contents := make([]string, 0) + for { + token := tokenizer.Next() + if token == html.ErrorToken { + break + } + if token == html.TextToken { + content := strings.TrimSpace(html.UnescapeString(string(tokenizer.Text()))) + if len(content) > 0 { + contents = append(contents, content) + } + } + } + return strings.Join(contents, " ") +} + +func (s *Storage) SyncSearch() { + rows, err := s.db.Query(` + select id, title, content, description + from items + where search_rowid is null; + `) + if err != nil { + s.log.Print(err) + return + } + + items := make([]Item, 0) + for rows.Next() { + var item Item + rows.Scan(&item.Id, &item.Title, &item.Content, &item.Description) + fmt.Println(item) + items = append(items, item) + } + + for _, item := range items { + result, err := s.db.Exec(` + insert into search (title, description, content) values (?, ?, ?)`, + item.Title, HTMLText(item.Description), HTMLText(item.Content), + ) + if err != nil { + s.log.Print(err) + return + } + if numrows, err := result.RowsAffected(); err == nil && numrows == 1 { + if rowId, err := result.LastInsertId(); err == nil { + s.db.Exec( + `update items set search_rowid = ? where id = ?`, + rowId, item.Id, + ) + } + } + } +}