james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-09-14 10:20:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix user agent using hardcoded 1.0 version.

Browse Source
This commit is contained in:
Aidan Holm
2024-10-26 18:56:26 +08:00
parent 0cef51c6ac
commit e81c076dbc
8 changed files with 38 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/yarr/main.go
View File

@@ -13,6 +13,7 @@ import (
"github.com/nkanaev/yarr/src/platform"
"github.com/nkanaev/yarr/src/server"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/worker"
)
var Version string = "0.0"
@@ -131,7 +132,8 @@ func main() {
log.Fatal("Failed to initialise database: ", err)
}
srv := server.NewServer(store, addr)
client := worker.NewClient(Version)
srv := server.NewServer(store, addr, client)
if basepath != "" {
srv.BasePath = "/" + strings.Trim(basepath, "/")

1
doc/changelog.txt
View File

@@ -10,6 +10,7 @@
- (fix) sorting same-day batch articles (thanks to @lamescholar for the report)
- (fix) showing login page in the selected theme (thanks to @feddiriko for the report)
- (fix) parsing atom feeds with html elements (thanks to @tillcash & @toBeOfUse for the report, @krkk for the fix)
- (fix) user agent now uses the current version (thanks to @aidanholm)
# v2.4 (2023-08-15)

4
src/server/routes.go
View File

@@ -230,7 +230,7 @@ func (s *Server) handleFeedList(c *router.Context) {
return
}
result, err := worker.DiscoverFeed(form.Url)
result, err := s.worker.DiscoverFeed(form.Url)
switch {
case err != nil:
log.Printf("Faild to discover feed for %s: %s", form.Url, err)
@@ -504,7 +504,7 @@ func (s *Server) handlePageCrawl(c *router.Context) {
return
}
body, err := worker.GetBody(url)
body, err := s.worker.GetBody(url)
if err != nil {
log.Print(err)
c.Out.WriteHeader(http.StatusBadRequest)

16
src/server/routes_test.go
View File

@@ -11,10 +11,12 @@ import (
"testing"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/worker"
)
func TestStatic(t *testing.T) {
handler := NewServer(nil, "127.0.0.1:8000").handler()
client := worker.NewClient("test")
handler := NewServer(nil, "127.0.0.1:8000", client).handler()
url := "/static/javascripts/app.js"
recorder := httptest.NewRecorder()
@@ -26,7 +28,8 @@ func TestStatic(t *testing.T) {
}
func TestStaticWithBase(t *testing.T) {
server := NewServer(nil, "127.0.0.1:8000")
client := worker.NewClient("test")
server := NewServer(nil, "127.0.0.1:8000", client)
server.BasePath = "/sub"
handler := server.handler()
@@ -41,7 +44,8 @@ func TestStaticWithBase(t *testing.T) {
}
func TestStaticBanTemplates(t *testing.T) {
handler := NewServer(nil, "127.0.0.1:8000").handler()
client := worker.NewClient("test")
handler := NewServer(nil, "127.0.0.1:8000", client).handler()
url := "/static/login.html"
recorder := httptest.NewRecorder()
@@ -56,7 +60,8 @@ func TestIndexGzipped(t *testing.T) {
log.SetOutput(io.Discard)
db, _ := storage.New(":memory:")
log.SetOutput(os.Stderr)
handler := NewServer(db, "127.0.0.1:8000").handler()
client := worker.NewClient("test")
handler := NewServer(db, "127.0.0.1:8000", client).handler()
url := "/"
recorder := httptest.NewRecorder()
@@ -87,7 +92,8 @@ func TestFeedIcons(t *testing.T) {
url := fmt.Sprintf("/api/feeds/%d/icon", feed.Id)
request := httptest.NewRequest("GET", url, nil)
handler := NewServer(db, "127.0.0.1:8000").handler()
client := worker.NewClient("test")
handler := NewServer(db, "127.0.0.1:8000", client).handler()
handler.ServeHTTP(recorder, request)
response := recorder.Result()

4
src/server/server.go
View File

@@ -26,11 +26,11 @@ type Server struct {
KeyFile string
}
func NewServer(db *storage.Storage, addr string) *Server {
func NewServer(db *storage.Storage, addr string, client *worker.Client) *Server {
return &Server{
db: db,
Addr: addr,
worker: worker.NewWorker(db),
worker: worker.NewWorker(db, client),
cache: make(map[string]interface{}),
cache_mutex: &sync.Mutex{},
}

9
src/worker/client.go
View File

@@ -30,9 +30,7 @@ func (c *Client) getConditional(url, lastModified, etag string) (*http.Response,
return c.httpClient.Do(req)
}
var client *Client
func init() {
func NewClient(version string) *Client {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
@@ -45,8 +43,9 @@ func init() {
Timeout: time.Second * 30,
Transport: transport,
}
client = &Client{
return &Client{
httpClient: httpClient,
userAgent: "Yarr/1.0",
userAgent: "Yarr/" + version,
}
}

20
src/worker/crawler.go
View File

@@ -28,10 +28,10 @@ type DiscoverResult struct {
Sources []FeedSource
}
func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
func (worker *Worker) DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
result := &DiscoverResult{}
// Query URL
res, err := client.get(candidateUrl)
res, err := worker.client.get(candidateUrl)
if err != nil {
return nil, err
}
@@ -74,7 +74,7 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
if sources[0].Url == candidateUrl {
return nil, errors.New("Recursion!")
}
return DiscoverFeed(sources[0].Url)
return worker.DiscoverFeed(sources[0].Url)
}
result.Sources = sources
@@ -89,7 +89,7 @@ var imageTypes = map[string]bool{
"image/gif": true,
}
func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
func (worker *Worker) findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
urls := make([]string, 0)
favicon := func(link string) string {
@@ -101,7 +101,7 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
}
if siteUrl != "" {
if res, err := client.get(siteUrl); err == nil {
if res, err := worker.client.get(siteUrl); err == nil {
defer res.Body.Close()
if body, err := ioutil.ReadAll(res.Body); err == nil {
urls = append(urls, scraper.FindIcons(string(body), siteUrl)...)
@@ -117,7 +117,7 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
}
for _, u := range urls {
res, err := client.get(u)
res, err := worker.client.get(u)
if err != nil {
continue
}
@@ -166,7 +166,7 @@ func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
return result
}
func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
func (worker *Worker) listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
lmod := ""
etag := ""
if state := db.GetHTTPState(f.Id); state != nil {
@@ -174,7 +174,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
etag = state.Etag
}
res, err := client.getConditional(f.FeedLink, lmod, etag)
res, err := worker.client.getConditional(f.FeedLink, lmod, etag)
if err != nil {
return nil, err
}
@@ -215,8 +215,8 @@ func getCharset(res *http.Response) string {
return ""
}
func GetBody(url string) (string, error) {
res, err := client.get(url)
func (worker *Worker) GetBody(url string) (string, error) {
res, err := worker.client.get(url)
if err != nil {
return "", err
}

9
src/worker/worker.go
View File

@@ -13,15 +13,16 @@ const NUM_WORKERS = 4
type Worker struct {
db *storage.Storage
client *Client
pending *int32
refresh *time.Ticker
reflock sync.Mutex
stopper chan bool
}
func NewWorker(db *storage.Storage) *Worker {
func NewWorker(db *storage.Storage, client *Client) *Worker {
pending := int32(0)
return &Worker{db: db, pending: &pending}
return &Worker{db: db, client: client, pending: &pending}
}
func (w *Worker) FeedsPending() int32 {
@@ -48,7 +49,7 @@ func (w *Worker) FindFavicons() {
}
func (w *Worker) FindFeedFavicon(feed storage.Feed) {
icon, err := findFavicon(feed.Link, feed.FeedLink)
icon, err := w.findFavicon(feed.Link, feed.FeedLink)
if err != nil {
log.Printf("Failed to find favicon for %s (%s): %s", feed.FeedLink, feed.Link, err)
}
@@ -137,7 +138,7 @@ func (w *Worker) refresher(feeds []storage.Feed) {
func (w *Worker) worker(srcqueue <-chan storage.Feed, dstqueue chan<- []storage.Item) {
for feed := range srcqueue {
items, err := listItems(feed, w.db)
items, err := w.listItems(feed, w.db)
if err != nil {
w.db.SetFeedError(feed.Id, err)
}