james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

do not convert response to utf8 if charset is not set

Browse Source
This commit is contained in:
Nazar Kanaev 2021-06-29 16:42:48 +01:00
parent 0c5385cef3
commit 982c4ebbbc
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/worker/crawler.go
View File

@ -4,9 +4,11 @@ import (
"bytes" "bytes"
"errors" "errors"
"fmt" "fmt"
"io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"strings"
"time" "time"
"github.com/nkanaev/yarr/src/content/scraper" "github.com/nkanaev/yarr/src/content/scraper"
@ -38,7 +40,7 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
return nil, fmt.Errorf("status code %d", res.StatusCode) return nil, fmt.Errorf("status code %d", res.StatusCode)
} }
body, err := charset.NewReader(res.Body, res.Header.Get("Content-Type")) body, err := httpBody(res)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -185,7 +187,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
return nil, nil return nil, nil
} }
body, err := charset.NewReader(res.Body, res.Header.Get("Content-Type")) body, err := httpBody(res)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -204,3 +206,11 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
feed.SetMissingDatesTo(time.Now()) feed.SetMissingDatesTo(time.Now())
return ConvertItems(feed.Items, f), nil return ConvertItems(feed.Items, f), nil
} }
func httpBody(res *http.Response) (io.Reader, error) {
ctype := res.Header.Get("Content-Type")
if strings.Contains(ctype, "charset") {
return charset.NewReader(res.Body, ctype)
}
return res.Body, nil
}