rewrite background jobs

server/handlers.go

@@ -62,7 +62,7 @@ func StaticHandler(rw http.ResponseWriter, req *http.Request) {
 
 func StatusHandler(rw http.ResponseWriter, req *http.Request) {
 	writeJSON(rw, map[string]interface{}{
-		"running": handler(req).fetchRunning,
+		"running": handler(req).queueSize > 0,
 		"stats":   db(req).FeedStats(),
 	})
 }

server/server.go

@@ -5,23 +5,24 @@ import (
 	"github.com/nkanaev/yarr/storage"
 	"log"
 	"net/http"
+	"runtime"
+	"sync/atomic"
+	"time"
 )
 
 type Handler struct {
 	db           *storage.Storage
 	log          *log.Logger
-	fetchRunning bool
 	feedQueue    chan storage.Feed
-	counter      chan int
+	queueSize    int32
-	queueSize    int
 }
 
 func New(db *storage.Storage, logger *log.Logger) *Handler {
 	return &Handler{
 		db:        db,
 		log:       logger,
-		feedQueue: make(chan storage.Feed),
+		feedQueue: make(chan storage.Feed, 1000),
-		counter:   make(chan int),
+		queueSize: 0,
 	}
 }
 
@@ -43,35 +44,41 @@ func (h Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 }
 
 func (h *Handler) startJobs() {
-	h.db.DeleteOldItems()
+	delTicker := time.NewTicker(time.Hour * 24)
-	go func() {
+	worker := func() {
 		for {
-			feed := <-h.feedQueue
+			select {
-			items := listItems(feed)
+			case feed := <-h.feedQueue:
-			h.db.CreateItems(items)
+				items := listItems(feed)
+				h.db.CreateItems(items)
+				atomic.AddInt32(&h.queueSize, -1)
+			case <- delTicker.C:
+				h.db.DeleteOldItems()
+			}
 		}
-	}()
+	}
-	go func() {
+
-		for {
+	num := runtime.NumCPU() - 1
-			val := <-h.counter
+	if num < 1 {
-			h.queueSize += val
+		num = 1
-		}
+	}
-	}()
+	for i := 0; i < num; i++ {
+		go worker()
+	}
+	go h.db.DeleteOldItems()
 	go h.db.SyncSearch()
-	h.fetchAllFeeds()
-}
 
-func (h *Handler) fetchFeed(feed storage.Feed) {
+	// fetch all feeds
-	h.queueSize += 1
-	h.feedQueue <- feed
-}
-
-func (h *Handler) fetchAllFeeds() {
 	for _, feed := range h.db.ListFeeds() {
 		h.fetchFeed(feed)
 	}
 }
 
+func (h *Handler) fetchFeed(feed storage.Feed) {
+	atomic.AddInt32(&h.queueSize, 1)
+	h.feedQueue <- feed
+}
+
 func Vars(req *http.Request) map[string]string {
 	if rv := req.Context().Value(ctxVars); rv != nil {
 		return rv.(map[string]string)

template/index.html

@@ -196,7 +196,7 @@
                     <div class="mt-4" v-if="feedNewChoice.length">
                         <p class="mb-2">
                             Multiple feeds found. Choose one below:
-                            <a href="#" class="float-right" @click.prevent="resetFeedChoice()">cancel</a>
+                            <a href="#" class="float-right text-decoration-none" @click.prevent="resetFeedChoice()">cancel</a>
                         </p>
                         <label class="selectgroup" v-for="choice in feedNewChoice">
                             <input type="radio" name="feedToAdd" :value="choice.url" v-model="feedNewChoiceSelected">