james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-24 00:33:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

do not convert response to utf8 if charset is not set

Browse Source
This commit is contained in:
Nazar Kanaev 2021-06-29 16:42:48 +01:00
parent 0c5385cef3
commit 982c4ebbbc
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/worker/crawler.go
View File

@ -4,9 +4,11 @@ import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
"github.com/nkanaev/yarr/src/content/scraper"
@ -38,7 +40,7 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
return nil, fmt.Errorf("status code %d", res.StatusCode)
}
body, err := charset.NewReader(res.Body, res.Header.Get("Content-Type"))
body, err := httpBody(res)
if err != nil {
return nil, err
}
@ -185,7 +187,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
return nil, nil
}
body, err := charset.NewReader(res.Body, res.Header.Get("Content-Type"))
body, err := httpBody(res)
if err != nil {
return nil, err
}
@ -204,3 +206,11 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
feed.SetMissingDatesTo(time.Now())
return ConvertItems(feed.Items, f), nil
}
func httpBody(res *http.Response) (io.Reader, error) {
ctype := res.Header.Get("Content-Type")
if strings.Contains(ctype, "charset") {
return charset.NewReader(res.Body, ctype)
}
return res.Body, nil
}