From e81c076dbcc84c57f671141e621fa747ce460c40 Mon Sep 17 00:00:00 2001 From: Aidan Holm Date: Sat, 26 Oct 2024 18:56:26 +0800 Subject: [PATCH] Fix user agent using hardcoded 1.0 version. --- cmd/yarr/main.go | 4 +++- doc/changelog.txt | 1 + src/server/routes.go | 4 ++-- src/server/routes_test.go | 16 +++++++++++----- src/server/server.go | 4 ++-- src/worker/client.go | 9 ++++----- src/worker/crawler.go | 20 ++++++++++---------- src/worker/worker.go | 9 +++++---- 8 files changed, 38 insertions(+), 29 deletions(-) diff --git a/cmd/yarr/main.go b/cmd/yarr/main.go index dbb4680..5a306cd 100644 --- a/cmd/yarr/main.go +++ b/cmd/yarr/main.go @@ -13,6 +13,7 @@ import ( "github.com/nkanaev/yarr/src/platform" "github.com/nkanaev/yarr/src/server" "github.com/nkanaev/yarr/src/storage" + "github.com/nkanaev/yarr/src/worker" ) var Version string = "0.0" @@ -131,7 +132,8 @@ func main() { log.Fatal("Failed to initialise database: ", err) } - srv := server.NewServer(store, addr) + client := worker.NewClient(Version) + srv := server.NewServer(store, addr, client) if basepath != "" { srv.BasePath = "/" + strings.Trim(basepath, "/") diff --git a/doc/changelog.txt b/doc/changelog.txt index ebd9075..94a2891 100644 --- a/doc/changelog.txt +++ b/doc/changelog.txt @@ -10,6 +10,7 @@ - (fix) sorting same-day batch articles (thanks to @lamescholar for the report) - (fix) showing login page in the selected theme (thanks to @feddiriko for the report) - (fix) parsing atom feeds with html elements (thanks to @tillcash & @toBeOfUse for the report, @krkk for the fix) +- (fix) user agent now uses the current version (thanks to @aidanholm) # v2.4 (2023-08-15) diff --git a/src/server/routes.go b/src/server/routes.go index 6d05320..a0d9e9a 100644 --- a/src/server/routes.go +++ b/src/server/routes.go @@ -230,7 +230,7 @@ func (s *Server) handleFeedList(c *router.Context) { return } - result, err := worker.DiscoverFeed(form.Url) + result, err := s.worker.DiscoverFeed(form.Url) switch { case err != nil: log.Printf("Faild to discover feed for %s: %s", form.Url, err) @@ -504,7 +504,7 @@ func (s *Server) handlePageCrawl(c *router.Context) { return } - body, err := worker.GetBody(url) + body, err := s.worker.GetBody(url) if err != nil { log.Print(err) c.Out.WriteHeader(http.StatusBadRequest) diff --git a/src/server/routes_test.go b/src/server/routes_test.go index cb53bb7..83c1465 100644 --- a/src/server/routes_test.go +++ b/src/server/routes_test.go @@ -11,10 +11,12 @@ import ( "testing" "github.com/nkanaev/yarr/src/storage" + "github.com/nkanaev/yarr/src/worker" ) func TestStatic(t *testing.T) { - handler := NewServer(nil, "127.0.0.1:8000").handler() + client := worker.NewClient("test") + handler := NewServer(nil, "127.0.0.1:8000", client).handler() url := "/static/javascripts/app.js" recorder := httptest.NewRecorder() @@ -26,7 +28,8 @@ func TestStatic(t *testing.T) { } func TestStaticWithBase(t *testing.T) { - server := NewServer(nil, "127.0.0.1:8000") + client := worker.NewClient("test") + server := NewServer(nil, "127.0.0.1:8000", client) server.BasePath = "/sub" handler := server.handler() @@ -41,7 +44,8 @@ func TestStaticWithBase(t *testing.T) { } func TestStaticBanTemplates(t *testing.T) { - handler := NewServer(nil, "127.0.0.1:8000").handler() + client := worker.NewClient("test") + handler := NewServer(nil, "127.0.0.1:8000", client).handler() url := "/static/login.html" recorder := httptest.NewRecorder() @@ -56,7 +60,8 @@ func TestIndexGzipped(t *testing.T) { log.SetOutput(io.Discard) db, _ := storage.New(":memory:") log.SetOutput(os.Stderr) - handler := NewServer(db, "127.0.0.1:8000").handler() + client := worker.NewClient("test") + handler := NewServer(db, "127.0.0.1:8000", client).handler() url := "/" recorder := httptest.NewRecorder() @@ -87,7 +92,8 @@ func TestFeedIcons(t *testing.T) { url := fmt.Sprintf("/api/feeds/%d/icon", feed.Id) request := httptest.NewRequest("GET", url, nil) - handler := NewServer(db, "127.0.0.1:8000").handler() + client := worker.NewClient("test") + handler := NewServer(db, "127.0.0.1:8000", client).handler() handler.ServeHTTP(recorder, request) response := recorder.Result() diff --git a/src/server/server.go b/src/server/server.go index d48f0a5..1e6f3e0 100644 --- a/src/server/server.go +++ b/src/server/server.go @@ -26,11 +26,11 @@ type Server struct { KeyFile string } -func NewServer(db *storage.Storage, addr string) *Server { +func NewServer(db *storage.Storage, addr string, client *worker.Client) *Server { return &Server{ db: db, Addr: addr, - worker: worker.NewWorker(db), + worker: worker.NewWorker(db, client), cache: make(map[string]interface{}), cache_mutex: &sync.Mutex{}, } diff --git a/src/worker/client.go b/src/worker/client.go index d7cac5d..7c4cef3 100644 --- a/src/worker/client.go +++ b/src/worker/client.go @@ -30,9 +30,7 @@ func (c *Client) getConditional(url, lastModified, etag string) (*http.Response, return c.httpClient.Do(req) } -var client *Client - -func init() { +func NewClient(version string) *Client { transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ @@ -45,8 +43,9 @@ func init() { Timeout: time.Second * 30, Transport: transport, } - client = &Client{ + + return &Client{ httpClient: httpClient, - userAgent: "Yarr/1.0", + userAgent: "Yarr/" + version, } } diff --git a/src/worker/crawler.go b/src/worker/crawler.go index 3720a38..e5e5ca4 100644 --- a/src/worker/crawler.go +++ b/src/worker/crawler.go @@ -28,10 +28,10 @@ type DiscoverResult struct { Sources []FeedSource } -func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) { +func (worker *Worker) DiscoverFeed(candidateUrl string) (*DiscoverResult, error) { result := &DiscoverResult{} // Query URL - res, err := client.get(candidateUrl) + res, err := worker.client.get(candidateUrl) if err != nil { return nil, err } @@ -74,7 +74,7 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) { if sources[0].Url == candidateUrl { return nil, errors.New("Recursion!") } - return DiscoverFeed(sources[0].Url) + return worker.DiscoverFeed(sources[0].Url) } result.Sources = sources @@ -89,7 +89,7 @@ var imageTypes = map[string]bool{ "image/gif": true, } -func findFavicon(siteUrl, feedUrl string) (*[]byte, error) { +func (worker *Worker) findFavicon(siteUrl, feedUrl string) (*[]byte, error) { urls := make([]string, 0) favicon := func(link string) string { @@ -101,7 +101,7 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) { } if siteUrl != "" { - if res, err := client.get(siteUrl); err == nil { + if res, err := worker.client.get(siteUrl); err == nil { defer res.Body.Close() if body, err := ioutil.ReadAll(res.Body); err == nil { urls = append(urls, scraper.FindIcons(string(body), siteUrl)...) @@ -117,7 +117,7 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) { } for _, u := range urls { - res, err := client.get(u) + res, err := worker.client.get(u) if err != nil { continue } @@ -166,7 +166,7 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item { return result } -func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) { +func (worker *Worker) listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) { lmod := "" etag := "" if state := db.GetHTTPState(f.Id); state != nil { @@ -174,7 +174,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) { etag = state.Etag } - res, err := client.getConditional(f.FeedLink, lmod, etag) + res, err := worker.client.getConditional(f.FeedLink, lmod, etag) if err != nil { return nil, err } @@ -215,8 +215,8 @@ func getCharset(res *http.Response) string { return "" } -func GetBody(url string) (string, error) { - res, err := client.get(url) +func (worker *Worker) GetBody(url string) (string, error) { + res, err := worker.client.get(url) if err != nil { return "", err } diff --git a/src/worker/worker.go b/src/worker/worker.go index bdf066e..4e75292 100644 --- a/src/worker/worker.go +++ b/src/worker/worker.go @@ -13,15 +13,16 @@ const NUM_WORKERS = 4 type Worker struct { db *storage.Storage + client *Client pending *int32 refresh *time.Ticker reflock sync.Mutex stopper chan bool } -func NewWorker(db *storage.Storage) *Worker { +func NewWorker(db *storage.Storage, client *Client) *Worker { pending := int32(0) - return &Worker{db: db, pending: &pending} + return &Worker{db: db, client: client, pending: &pending} } func (w *Worker) FeedsPending() int32 { @@ -48,7 +49,7 @@ func (w *Worker) FindFavicons() { } func (w *Worker) FindFeedFavicon(feed storage.Feed) { - icon, err := findFavicon(feed.Link, feed.FeedLink) + icon, err := w.findFavicon(feed.Link, feed.FeedLink) if err != nil { log.Printf("Failed to find favicon for %s (%s): %s", feed.FeedLink, feed.Link, err) } @@ -137,7 +138,7 @@ func (w *Worker) refresher(feeds []storage.Feed) { func (w *Worker) worker(srcqueue <-chan storage.Feed, dstqueue chan<- []storage.Item) { for feed := range srcqueue { - items, err := listItems(feed, w.db) + items, err := w.listItems(feed, w.db) if err != nil { w.db.SetFeedError(feed.Id, err) }