From 80402943a132039ee478f0bda721e0ebce94bbe2 Mon Sep 17 00:00:00 2001
From: Nazar Kanaev <nkanaev@live.com>
Date: Wed, 24 Mar 2021 13:42:24 +0000
Subject: [PATCH] wrap in charset

---
 src/worker/crawler.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/worker/crawler.go b/src/worker/crawler.go
index c5d63d4..7a6fb4f 100644
--- a/src/worker/crawler.go
+++ b/src/worker/crawler.go
@@ -36,7 +36,12 @@ func DiscoverFeed(candidateUrl string) (*DiscoverResult, error) {
 	if res.StatusCode != 200 {
 		return nil, fmt.Errorf("status code %d", res.StatusCode)
 	}
-	content, err := ioutil.ReadAll(res.Body)
+
+	body, err := charset.NewReader(res.Body, res.Header.Get("Content-Type"))
+	if err != nil {
+		return nil, err
+	}
+	content, err := ioutil.ReadAll(body)
 	if err != nil {
 		return nil, err
 	}