Merge e81c076dbc into 0cef51c6ac

cmd/yarr/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/nkanaev/yarr/src/platform"
 	"github.com/nkanaev/yarr/src/server"
 	"github.com/nkanaev/yarr/src/storage"
+	"github.com/nkanaev/yarr/src/worker"
 )
 
 var Version string = "0.0"
@@ -131,7 +132,8 @@ func main() {
 		log.Fatal("Failed to initialise database: ", err)
 	}
 
-	srv := server.NewServer(store, addr)
+	client := worker.NewClient(Version)
+	srv := server.NewServer(store, addr, client)
 
 	if basepath != "" {
 		srv.BasePath = "/" + strings.Trim(basepath, "/")

doc/changelog.txt

@@ -10,6 +10,7 @@
 - (fix) sorting same-day batch articles (thanks to @lamescholar for the report)
 - (fix) showing login page in the selected theme (thanks to @feddiriko for the report)
 - (fix) parsing atom feeds with html elements (thanks to @tillcash & @toBeOfUse for the report, @krkk for the fix)
+- (fix) user agent now uses the current version (thanks to @aidanholm)
 
 # v2.4 (2023-08-15)
 

src/server/routes.go

@@ -230,7 +230,7 @@ func (s *Server) handleFeedList(c *router.Context) {
 			return
 		}
 
-		result, err := worker.DiscoverFeed(form.Url)
+		result, err := s.worker.DiscoverFeed(form.Url)
 		switch {
 		case err != nil:
 			log.Printf("Faild to discover feed for %s: %s", form.Url, err)
@@ -504,7 +504,7 @@ func (s *Server) handlePageCrawl(c *router.Context) {
 		return
 	}
 
-	body, err := worker.GetBody(url)
+	body, err := s.worker.GetBody(url)
 	if err != nil {
 		log.Print(err)
 		c.Out.WriteHeader(http.StatusBadRequest)

src/server/routes_test.go

@@ -11,10 +11,12 @@ import (
 	"testing"
 
 	"github.com/nkanaev/yarr/src/storage"
+	"github.com/nkanaev/yarr/src/worker"
 )
 
 func TestStatic(t *testing.T) {
-	handler := NewServer(nil, "127.0.0.1:8000").handler()
+	client := worker.NewClient("test")
+	handler := NewServer(nil, "127.0.0.1:8000", client).handler()
 	url := "/static/javascripts/app.js"
 
 	recorder := httptest.NewRecorder()
@@ -26,7 +28,8 @@ func TestStatic(t *testing.T) {
 }
 
 func TestStaticWithBase(t *testing.T) {
-	server := NewServer(nil, "127.0.0.1:8000")
+	client := worker.NewClient("test")
+	server := NewServer(nil, "127.0.0.1:8000", client)
 	server.BasePath = "/sub"
 
 	handler := server.handler()
@@ -41,7 +44,8 @@ func TestStaticWithBase(t *testing.T) {
 }
 
 func TestStaticBanTemplates(t *testing.T) {
-	handler := NewServer(nil, "127.0.0.1:8000").handler()
+	client := worker.NewClient("test")
+	handler := NewServer(nil, "127.0.0.1:8000", client).handler()
 	url := "/static/login.html"
 
 	recorder := httptest.NewRecorder()
@@ -56,7 +60,8 @@ func TestIndexGzipped(t *testing.T) {
 	log.SetOutput(io.Discard)
 	db, _ := storage.New(":memory:")
 	log.SetOutput(os.Stderr)
-	handler := NewServer(db, "127.0.0.1:8000").handler()
+	client := worker.NewClient("test")
+	handler := NewServer(db, "127.0.0.1:8000", client).handler()
 	url := "/"
 
 	recorder := httptest.NewRecorder()
@@ -87,7 +92,8 @@ func TestFeedIcons(t *testing.T) {
 	url := fmt.Sprintf("/api/feeds/%d/icon", feed.Id)
 	request := httptest.NewRequest("GET", url, nil)
 
-	handler := NewServer(db, "127.0.0.1:8000").handler()
+	client := worker.NewClient("test")
+	handler := NewServer(db, "127.0.0.1:8000", client).handler()
 	handler.ServeHTTP(recorder, request)
 	response := recorder.Result()
 

src/server/server.go

@@ -26,11 +26,11 @@ type Server struct {
 	KeyFile  string
 }
 
-func NewServer(db *storage.Storage, addr string) *Server {
+func NewServer(db *storage.Storage, addr string, client *worker.Client) *Server {
 	return &Server{
 		db:          db,
 		Addr:        addr,
-		worker:      worker.NewWorker(db),
+		worker:      worker.NewWorker(db, client),
 		cache:       make(map[string]interface{}),
 		cache_mutex: &sync.Mutex{},
 	}

src/worker/client.go

@@ -30,9 +30,7 @@ func (c *Client) getConditional(url, lastModified, etag string) (*http.Response,
 	return c.httpClient.Do(req)
 }
 
-var client *Client
+func NewClient(version string) *Client {
-
-func init() {
 	transport := &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{
@@ -45,8 +43,9 @@ func init() {
 		Timeout:   time.Second * 30,
 		Transport: transport,
 	}
-	client = &Client{
+
+	return &Client{
 		httpClient: httpClient,
-		userAgent:  "Yarr/1.0",
+		userAgent:  "Yarr/" + version,
 	}
 }

src/worker/crawler.go

@@ -28,10 +28,10 @@ type DiscoverResult struct {
 	Sources  []FeedSource
 }
 
-func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
+func (worker *Worker) DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
 	result := &DiscoverResult{}
 	// Query URL
-	res, err := client.get(candidateUrl)
+	res, err := worker.client.get(candidateUrl)
 	if err != nil {
 		return nil, err
 	}
@@ -74,7 +74,7 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
 		if sources[0].Url == candidateUrl {
 			return nil, errors.New("Recursion!")
 		}
-		return DiscoverFeed(sources[0].Url)
+		return worker.DiscoverFeed(sources[0].Url)
 	}
 
 	result.Sources = sources
@@ -89,7 +89,7 @@ var imageTypes = map[string]bool{
 	"image/gif":    true,
 }
 
-func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
+func (worker *Worker) findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
 	urls := make([]string, 0)
 
 	favicon := func(link string) string {
@@ -101,7 +101,7 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
 	}
 
 	if siteUrl != "" {
-		if res, err := client.get(siteUrl); err == nil {
+		if res, err := worker.client.get(siteUrl); err == nil {
 			defer res.Body.Close()
 			if body, err := ioutil.ReadAll(res.Body); err == nil {
 				urls = append(urls, scraper.FindIcons(string(body), siteUrl)...)
@@ -117,7 +117,7 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
 	}
 
 	for _, u := range urls {
-		res, err := client.get(u)
+		res, err := worker.client.get(u)
 		if err != nil {
 			continue
 		}
@@ -166,7 +166,7 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
 	return result
 }
 
-func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
+func (worker *Worker) listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 	lmod := ""
 	etag := ""
 	if state := db.GetHTTPState(f.Id); state != nil {
@@ -174,7 +174,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
 		etag = state.Etag
 	}
 
-	res, err := client.getConditional(f.FeedLink, lmod, etag)
+	res, err := worker.client.getConditional(f.FeedLink, lmod, etag)
 	if err != nil {
 		return nil, err
 	}
@@ -215,8 +215,8 @@ func getCharset(res *http.Response) string {
 	return ""
 }
 
-func GetBody(url string) (string, error) {
+func (worker *Worker) GetBody(url string) (string, error) {
-	res, err := client.get(url)
+	res, err := worker.client.get(url)
 	if err != nil {
 		return "", err
 	}

src/worker/worker.go

@@ -13,15 +13,16 @@ const NUM_WORKERS = 4
 
 type Worker struct {
 	db      *storage.Storage
+	client  *Client
 	pending *int32
 	refresh *time.Ticker
 	reflock sync.Mutex
 	stopper chan bool
 }
 
-func NewWorker(db *storage.Storage) *Worker {
+func NewWorker(db *storage.Storage, client *Client) *Worker {
 	pending := int32(0)
-	return &Worker{db: db, pending: &pending}
+	return &Worker{db: db, client: client, pending: &pending}
 }
 
 func (w *Worker) FeedsPending() int32 {
@@ -48,7 +49,7 @@ func (w *Worker) FindFavicons() {
 }
 
 func (w *Worker) FindFeedFavicon(feed storage.Feed) {
-	icon, err := findFavicon(feed.Link, feed.FeedLink)
+	icon, err := w.findFavicon(feed.Link, feed.FeedLink)
 	if err != nil {
 		log.Printf("Failed to find favicon for %s (%s): %s", feed.FeedLink, feed.Link, err)
 	}
@@ -137,7 +138,7 @@ func (w *Worker) refresher(feeds []storage.Feed) {
 
 func (w *Worker) worker(srcqueue <-chan storage.Feed, dstqueue chan<- []storage.Item) {
 	for feed := range srcqueue {
-		items, err := listItems(feed, w.db)
+		items, err := w.listItems(feed, w.db)
 		if err != nil {
 			w.db.SetFeedError(feed.Id, err)
 		}