james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-09-13 18:00:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix page crawling encoding

Browse Source
This commit is contained in:
nkanaev
2022-01-24 14:02:08 +00:00
parent b78c8bf8bf
commit e3e9542f1e
3 changed files with 20 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/worker/crawler.go
View File

@@ -202,10 +202,14 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
return ConvertItems(feed.Items, f), nil
}
func httpBody(res *http.Response) (io.Reader, error) {
func httpBody(res *http.Response) (io.ReadCloser, error) {
ctype := res.Header.Get("Content-Type")
if strings.Contains(ctype, "charset") {
return charset.NewReader(res.Body, ctype)
reader, err := charset.NewReader(res.Body, ctype)
if err != nil {
return nil, err
}
return io.NopCloser(reader), nil
}
return res.Body, nil
}