basic feed fetcher

server/handlers.go

@@ -174,6 +174,40 @@ func FeedListHandler(rw http.ResponseWriter, req *http.Request) {
 	}
 }
 
+func convertItems(items []*gofeed.Item, feed storage.Feed) []storage.Item {
+	result := make([]storage.Item, len(items))
+	for _, item := range items {
+		imageURL := ""
+		if item.Image != nil {
+			imageURL = item.Image.URL
+		}
+		author := ""
+		if item.Author != nil {
+			author = item.Author.Name
+		}
+		result = append(result, storage.Item{
+			Id: item.GUID,
+			FeedId: feed.Id,
+			Title: item.Title,
+			Link: item.Link,
+			Description: item.Description,
+			Content: item.Content,
+			Author: author,
+			Date: item.PublishedParsed,
+			DateUpdated: item.UpdatedParsed,
+			Status: storage.UNREAD,
+			Image: imageURL,
+		})
+	}
+	return result
+}
+
+func listItems(f storage.Feed) []storage.Item {
+	fp := gofeed.NewParser()	
+	feed, _ := fp.ParseURL(f.FeedLink)
+	return convertItems(feed.Items, f)
+}
+
 func createFeed(s *storage.Storage, url string, folderId *int64) error {
 	fp := gofeed.NewParser()
 	feed, err := fp.ParseURL(url)
@@ -184,7 +218,7 @@ func createFeed(s *storage.Storage, url string, folderId *int64) error {
 	if len(feedLink) == 0 {
 		feedLink = url
 	}
-	entry := s.CreateFeed(
+	s.CreateFeed(
 		feed.Title,
 		feed.Description,
 		feed.Link,
@@ -192,8 +226,6 @@ func createFeed(s *storage.Storage, url string, folderId *int64) error {
 		"",
 		folderId,
 	)
-
-	fmt.Println("here we go", entry)
 	return nil
 }
 

server/server.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"github.com/nkanaev/yarr/storage"
 	"log"
+	"fmt"
 )
 
 type Route struct {
@@ -18,6 +19,41 @@ type Route struct {
 type Handler struct {
 	db *storage.Storage
 	fetchRunning bool
+	feedQueue chan storage.Feed
+	counter chan int
+	queueSize int
+}
+
+func (h *Handler) startJobs() {
+	go func() {
+		fmt.Println("one")
+		for {
+			feed := <-h.feedQueue
+			fmt.Println("got feed", feed)
+			items := listItems(feed)
+			h.db.CreateItems(items)
+		}
+	}()
+	go func() {
+		fmt.Println("two")
+		for {
+			val := <-h.counter
+			h.queueSize += val
+		}			
+	}()
+	h.fetchAllFeeds()
+}
+
+func (h *Handler) fetchFeed(feed storage.Feed) {
+	h.queueSize += 1	
+	h.feedQueue <- feed
+}
+
+func (h *Handler) fetchAllFeeds() {
+	for _, feed := range h.db.ListFeeds() {
+		fmt.Println("here", feed)
+		h.fetchFeed(feed)
+	}
 }
 
 func p(path string, handler func(http.ResponseWriter, *http.Request)) Route {
@@ -103,7 +139,12 @@ func writeJSON(rw http.ResponseWriter, data interface{}) {
 
 func New() *http.Server {
 	db, _ := storage.New()
-	h := Handler{db: db}
+	h := Handler{
+		db: db,
+		feedQueue: make(chan storage.Feed),
+		counter: make(chan int),
+	}
 	s := &http.Server{Addr: "127.0.0.1:8000", Handler: h}
+	h.startJobs()
 	return s
 }

storage/item.go

@@ -1,6 +1,9 @@
 package storage
 
-import "fmt"
+import (
+	"fmt"
+	"time"
+)
 
 type ItemStatus int
 
@@ -18,8 +21,8 @@ type Item struct {
 	Description string
 	Content string
 	Author string
-	Date int64
+	Date *time.Time
-	DateUpdated int64
+	DateUpdated *time.Time
 	Status ItemStatus
 	Image string
 }

storage/storage.go

@@ -35,8 +35,8 @@ create table if not exists items (
  description    text,
  content        text,
  author         text,
- date           integer,
+ date           datetime,
- date_updated   integer,
+ date_updated   datetime,
  status         integer,
  image          text
 );