reorganizing content-related packages

2025-10-30 14:33:31 +00:00 · 2021-03-31 23:40:59 +01:00
parent 0b8bf50204
commit b04e8c1e93
12 changed files with 19 additions and 18 deletions
--- a/src/content/htmlutil/query.go
+++ b/src/content/htmlutil/query.go
@@ -0,0 +1,76 @@
+package htmlutil
+
+import (
+	"regexp"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+var nodeNameRegex = regexp.MustCompile(`\w+|\*`)
+
+func FindNodes(node *html.Node, match func(*html.Node) bool) []*html.Node {
+	nodes := make([]*html.Node, 0)
+
+	queue := make([]*html.Node, 0)
+	queue = append(queue, node)
+	for len(queue) > 0 {
+		var n *html.Node
+		n, queue = queue[0], queue[1:]
+		if match(n) {
+			nodes = append(nodes, n)
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			queue = append(queue, c)
+		}
+	}
+	return nodes
+}
+
+func Query(node *html.Node, sel string) []*html.Node {
+	matcher := NewMatcher(sel)
+	return FindNodes(node, matcher.Match)
+}
+
+func NewMatcher(sel string) Matcher {
+	multi := MultiMatch{}
+	parts := strings.Split(sel, ",")
+	for _, part := range parts {
+		part := strings.TrimSpace(part)
+		if nodeNameRegex.MatchString(part) {
+			multi.Add(ElementMatch{Name: part})
+		} else {
+			panic("unsupported selector: " + part)
+		}
+	}
+	return multi
+}
+
+type Matcher interface {
+	Match(*html.Node) bool
+}
+
+type ElementMatch struct {
+	Name string
+}
+
+func (m ElementMatch) Match(n *html.Node) bool {
+	return n.Type == html.ElementNode && (n.Data == m.Name || m.Name == "*")
+}
+
+type MultiMatch struct {
+	matchers []Matcher
+}
+
+func (m *MultiMatch) Add(matcher Matcher) {
+	m.matchers = append(m.matchers, matcher)
+}
+
+func (m MultiMatch) Match(n *html.Node) bool {
+	for _, matcher := range m.matchers {
+		if matcher.Match(n) {
+			return true
+		}
+	}
+	return false
+}
--- a/src/content/htmlutil/query_test.go
+++ b/src/content/htmlutil/query_test.go
@@ -0,0 +1,64 @@
+package htmlutil
+
+import (
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+func TestQuery(t *testing.T) {
+	node, _ := html.Parse(strings.NewReader(`
+		<!DOCTYPE html>
+		<html lang="en">
+		<head>
+			<meta charset="UTF-8">
+			<title></title>
+		</head>
+		<body>
+			<div>
+				<p>test</p>
+			</div>
+		</body>
+		</html>
+	`))
+	nodes := Query(node, "p")
+	match := (
+		len(nodes) == 1 &&
+		nodes[0].Type == html.ElementNode &&
+		nodes[0].Data == "p")		
+	if !match {
+		t.Fatalf("incorrect match: %#v", nodes)
+	}
+}
+
+func TestQueryMulti(t *testing.T) {
+	node, _ := html.Parse(strings.NewReader(`
+		<!DOCTYPE html>
+		<html lang="en">
+		<head>
+			<meta charset="UTF-8">
+			<title></title>
+		</head>
+		<body>
+			<p>foo</p>
+			<div>
+				<p>bar</p>
+				<span>baz</span>
+			</div>
+		</body>
+		</html>
+	`))
+	nodes := Query(node, "p , span")
+	match := (
+		len(nodes) == 3 &&
+		nodes[0].Type == html.ElementNode && nodes[0].Data == "p" &&
+		nodes[1].Type == html.ElementNode && nodes[1].Data == "p" &&
+		nodes[2].Type == html.ElementNode && nodes[2].Data == "span")	
+	if !match {
+		for i, n := range nodes {
+			t.Logf("%d: %s", i, HTML(n))
+		}
+		t.Fatal("incorrect match")
+	}
+}
--- a/src/content/htmlutil/urlutils.go
+++ b/src/content/htmlutil/urlutils.go
@@ -0,0 +1,33 @@
+package htmlutil
+
+import (
+	"net/url"
+)
+
+func Any(els []string, el string, match func(string, string) bool) bool {
+	for _, x := range els {
+		if match(x, el) {
+			return true
+		}
+	}
+	return false
+}
+
+func AbsoluteUrl(href, base string) string {
+	baseUrl, err := url.Parse(base)
+	if err != nil {
+		return ""
+	}
+	hrefUrl, err := url.Parse(href)
+	if err != nil {
+		return ""
+	}
+	return baseUrl.ResolveReference(hrefUrl).String()
+}
+
+func URLDomain(val string) string {
+	if u, err := url.Parse(val); err == nil {
+		return u.Host
+	}
+	return val
+}
--- a/src/content/htmlutil/utils.go
+++ b/src/content/htmlutil/utils.go
@@ -0,0 +1,41 @@
+package htmlutil
+
+import (
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+func HTML(node *html.Node) string {
+	writer := strings.Builder{}
+	html.Render(&writer, node)
+	return writer.String()
+}
+
+func InnerHTML(node *html.Node) string {
+	writer := strings.Builder{}
+	for c := node.FirstChild; c != nil; c = c.NextSibling {
+		html.Render(&writer, c)
+	}
+	return writer.String()
+}
+
+func Attr(node *html.Node, key string) string {
+	for _, a := range node.Attr {
+		if strings.EqualFold(a.Key, key) {
+			return a.Val
+		}
+	}
+	return ""
+}
+
+func Text(node *html.Node) string {
+	text := make([]string, 0)
+	isTextNode := func(n *html.Node) bool {
+		return n.Type == html.TextNode
+	}
+	for _, n := range FindNodes(node, isTextNode) {
+		text = append(text, strings.TrimSpace(n.Data))
+	}
+	return strings.Join(text, " ")
+}
--- a/src/content/readability/LICENSE
+++ b/src/content/readability/LICENSE
@@ -0,0 +1,177 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
--- a/src/content/readability/readability.go
+++ b/src/content/readability/readability.go
@@ -0,0 +1,271 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package readability
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	//"log"
+	"math"
+	"regexp"
+	"strings"
+
+	"github.com/nkanaev/yarr/src/content/htmlutil"
+	"golang.org/x/net/html"
+)
+
+const (
+	defaultTagsToScore = "section,h2,h3,h4,h5,h6,p,td,pre,div"
+)
+
+var (
+	divToPElementsRegexp = regexp.MustCompile(`(?i)<(a|blockquote|dl|div|img|ol|p|pre|table|ul)`)
+	sentenceRegexp       = regexp.MustCompile(`\.( |$)`)
+
+	blacklistCandidatesRegexp  = regexp.MustCompile(`(?i)popupbody|-ad|g-plus`)
+	okMaybeItsACandidateRegexp = regexp.MustCompile(`(?i)and|article|body|column|main|shadow`)
+	unlikelyCandidatesRegexp   = regexp.MustCompile(`(?i)banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`)
+
+	negativeRegexp = regexp.MustCompile(`(?i)hidden|^hid$|hid$|hid|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby|p-author`)
+	positiveRegexp = regexp.MustCompile(`(?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story`)
+)
+
+type nodeScores map[*html.Node]float32
+
+// ExtractContent returns relevant content.
+func ExtractContent(page io.Reader) (string, error) {
+	root, err := html.Parse(page)
+	if err != nil {
+		return "", err
+	}
+
+	for _, trash := range htmlutil.Query(root, "script,style") {
+		if trash.Parent != nil {
+			trash.Parent.RemoveChild(trash)
+		}
+	}
+
+	transformMisusedDivsIntoParagraphs(root)
+	removeUnlikelyCandidates(root)
+
+	scores := getCandidates(root)
+	//log.Printf("[Readability] Candidates: %v", candidates)
+
+	best := getTopCandidate(scores)
+	if best == nil {
+		best = root
+	}
+	//log.Printf("[Readability] TopCandidate: %v", topCandidate)
+
+	output := getArticle(best, scores)
+	return output, nil
+}
+
+// Now that we have the top candidate, look through its siblings for content that might also be related.
+// Things like preambles, content split by ads that we removed, etc.
+func getArticle(best *html.Node, scores nodeScores) string {
+	output := bytes.NewBufferString("<div>")
+	siblingScoreThreshold := float32(math.Max(10, float64(scores[best]*.2)))
+
+	nodelist := make([]*html.Node, 0)
+	nodelist = append(nodelist, best)
+
+	// Get the candidate's siblings
+	for n := best.NextSibling; n != nil; n = n.NextSibling {
+		nodelist = append(nodelist, n)
+	}
+	for n := best.PrevSibling; n != nil; n = n.PrevSibling {
+		nodelist = append(nodelist, n)
+	}
+
+	for _, node := range nodelist {
+		append := false
+		isP := node.Data == "p"
+
+		if node == best {
+			append = true
+		} else if scores[node] >= siblingScoreThreshold {
+			append = true
+		} else {
+			if isP {
+				linkDensity := getLinkDensity(node)
+				content := htmlutil.Text(node)
+				contentLength := len(content)
+
+				if contentLength >= 80 && linkDensity < .25 {
+					append = true
+				} else if contentLength < 80 && linkDensity == 0 && sentenceRegexp.MatchString(content) {
+					append = true
+				}
+			}
+		}
+		if append {
+			tag := "div"
+			if isP {
+				tag = "p"
+			}
+			fmt.Fprintf(output, "<%s>%s</%s>", tag, htmlutil.InnerHTML(node), tag)
+		}
+	}
+
+	output.Write([]byte("</div>"))
+	return output.String()
+}
+
+func removeUnlikelyCandidates(root *html.Node) {
+	body := htmlutil.Query(root, "body")
+	if len(body) == 0 {
+		return
+	}
+	for _, node := range htmlutil.Query(body[0], "*") {
+		str := htmlutil.Attr(node, "class") + htmlutil.Attr(node, "id")
+
+		blacklisted := (
+			blacklistCandidatesRegexp.MatchString(str) ||
+			(unlikelyCandidatesRegexp.MatchString(str) &&
+			 !okMaybeItsACandidateRegexp.MatchString(str)))
+		if blacklisted && node.Parent != nil {
+			node.Parent.RemoveChild(node)
+		}
+	}
+}
+
+func getTopCandidate(scores nodeScores) *html.Node {
+	var top *html.Node
+	var max float32
+
+	for node, score := range scores {
+		if score > max {
+			top = node
+			max = score
+		}
+	}
+
+	return top
+}
+
+// Loop through all paragraphs, and assign a score to them based on how content-y they look.
+// Then add their score to their parent node.
+// A score is determined by things like number of commas, class names, etc.
+// Maybe eventually link density.
+func getCandidates(root *html.Node) nodeScores {
+	scores := make(nodeScores)
+	for _, node := range htmlutil.Query(root, defaultTagsToScore) {
+		text := htmlutil.Text(node)
+
+		// If this paragraph is less than 25 characters, don't even count it.
+		if len(text) < 25 {
+			continue
+		}
+
+		parentNode := node.Parent
+		grandParentNode := parentNode.Parent
+
+		if _, found := scores[parentNode]; !found {
+			scores[parentNode] = scoreNode(parentNode)
+		}
+
+		if grandParentNode != nil {
+			if _, found := scores[grandParentNode]; !found {
+				scores[grandParentNode] = scoreNode(grandParentNode)
+			}
+		}
+
+		// Add a point for the paragraph itself as a base.
+		contentScore := float32(1.0)
+
+		// Add points for any commas within this paragraph.
+		contentScore += float32(strings.Count(text, ",") + 1)
+
+		// For every 100 characters in this paragraph, add another point. Up to 3 points.
+		contentScore += float32(math.Min(float64(int(len(text)/100.0)), 3))
+
+		scores[parentNode] += contentScore
+		if grandParentNode != nil {
+			scores[grandParentNode] += contentScore / 2.0
+		}
+	}
+
+	// Scale the final candidates score based on link density. Good content
+	// should have a relatively small link density (5% or less) and be mostly
+	// unaffected by this operation
+	for node, _ := range scores {
+		scores[node] *= (1 - getLinkDensity(node))
+	}
+
+	return scores
+}
+
+func scoreNode(node *html.Node) float32 {
+	var score float32
+
+	switch node.Data {
+	case "div":
+		score += 5
+	case "pre", "td", "blockquote", "img":
+		score += 3
+	case "address", "ol", "ul", "dl", "dd", "dt", "li", "form":
+		score -= 3
+	case "h1", "h2", "h3", "h4", "h5", "h6", "th":
+		score -= 5
+	}
+
+	return score + getClassWeight(node)
+}
+
+// Get the density of links as a percentage of the content
+// This is the amount of text that is inside a link divided by the total text in the node.
+func getLinkDensity(n *html.Node) float32 {
+	textLength := len(htmlutil.Text(n))
+	if textLength == 0 {
+		return 0
+	}
+
+	linkLength := 0.0
+	for _, a := range htmlutil.Query(n, "a") {
+		linkLength += float64(len(htmlutil.Text(a)))
+	}
+
+	return float32(linkLength) / float32(textLength)
+}
+
+// Get an elements class/id weight. Uses regular expressions to tell if this
+// element looks good or bad.
+func getClassWeight(node *html.Node) float32 {
+	weight := 0
+	class := htmlutil.Attr(node, "class")
+	id := htmlutil.Attr(node, "id")
+
+	if class != "" {
+		if negativeRegexp.MatchString(class) {
+			weight -= 25
+		}
+
+		if positiveRegexp.MatchString(class) {
+			weight += 25
+		}
+	}
+
+	if id != "" {
+		if negativeRegexp.MatchString(id) {
+			weight -= 25
+		}
+
+		if positiveRegexp.MatchString(id) {
+			weight += 25
+		}
+	}
+
+	return float32(weight)
+}
+
+func transformMisusedDivsIntoParagraphs(root *html.Node) {
+	for _, node := range htmlutil.Query(root, "div") {
+		if !divToPElementsRegexp.MatchString(htmlutil.InnerHTML(node)) {
+			node.Data = "p"
+		}
+	}
+}
--- a/src/content/sanitizer/sanitizer.go
+++ b/src/content/sanitizer/sanitizer.go
@@ -0,0 +1,469 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package scraper
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/nkanaev/yarr/src/content/htmlutil"
+	"golang.org/x/net/html"
+)
+
+var splitSrcsetRegex = regexp.MustCompile(`,\s+`)
+
+// Sanitize returns safe HTML.
+func Sanitize(baseURL, input string) string {
+	var buffer bytes.Buffer
+	var tagStack []string
+	var parentTag string
+	blacklistedTagDepth := 0
+
+	tokenizer := html.NewTokenizer(bytes.NewBufferString(input))
+	for {
+		if tokenizer.Next() == html.ErrorToken {
+			err := tokenizer.Err()
+			if err == io.EOF {
+				return buffer.String()
+			}
+
+			return ""
+		}
+
+		token := tokenizer.Token()
+		switch token.Type {
+		case html.TextToken:
+			if blacklistedTagDepth > 0 {
+				continue
+			}
+
+			// An iframe element never has fallback content.
+			// See https://www.w3.org/TR/2010/WD-html5-20101019/the-iframe-element.html#the-iframe-element
+			if parentTag == "iframe" {
+				continue
+			}
+
+			buffer.WriteString(html.EscapeString(token.Data))
+		case html.StartTagToken:
+			tagName := token.DataAtom.String()
+			parentTag = tagName
+
+			if isValidTag(tagName) {
+				attrNames, htmlAttributes := sanitizeAttributes(baseURL, tagName, token.Attr)
+
+				if hasRequiredAttributes(tagName, attrNames) {
+					if len(attrNames) > 0 {
+						buffer.WriteString("<" + tagName + " " + htmlAttributes + ">")
+					} else {
+						buffer.WriteString("<" + tagName + ">")
+					}
+
+					tagStack = append(tagStack, tagName)
+				}
+			} else if isBlockedTag(tagName) {
+				blacklistedTagDepth++
+			}
+		case html.EndTagToken:
+			tagName := token.DataAtom.String()
+			if isValidTag(tagName) && inList(tagName, tagStack) {
+				buffer.WriteString(fmt.Sprintf("</%s>", tagName))
+			} else if isBlockedTag(tagName) {
+				blacklistedTagDepth--
+			}
+		case html.SelfClosingTagToken:
+			tagName := token.DataAtom.String()
+			if isValidTag(tagName) {
+				attrNames, htmlAttributes := sanitizeAttributes(baseURL, tagName, token.Attr)
+
+				if hasRequiredAttributes(tagName, attrNames) {
+					if len(attrNames) > 0 {
+						buffer.WriteString("<" + tagName + " " + htmlAttributes + "/>")
+					} else {
+						buffer.WriteString("<" + tagName + "/>")
+					}
+				}
+			}
+		}
+	}
+}
+
+func sanitizeAttributes(baseURL, tagName string, attributes []html.Attribute) ([]string, string) {
+	var htmlAttrs, attrNames []string
+
+	for _, attribute := range attributes {
+		value := attribute.Val
+
+		if !isValidAttribute(tagName, attribute.Key) {
+			continue
+		}
+
+		if (tagName == "img" || tagName == "source") && attribute.Key == "srcset" {
+			value = sanitizeSrcsetAttr(baseURL, value)
+		}
+
+		if isExternalResourceAttribute(attribute.Key) {
+			if tagName == "iframe" {
+				if isValidIframeSource(baseURL, attribute.Val) {
+					value = attribute.Val
+				} else {
+					continue
+				}
+			} else if tagName == "img" && attribute.Key == "src" && isValidDataAttribute(attribute.Val) {
+				value = attribute.Val
+			} else {
+				value = htmlutil.AbsoluteUrl(value, baseURL)
+				if value == "" {
+					continue
+				}
+
+				if !hasValidURIScheme(value) || isBlockedResource(value) {
+					continue
+				}
+			}
+		}
+
+		attrNames = append(attrNames, attribute.Key)
+		htmlAttrs = append(htmlAttrs, fmt.Sprintf(`%s="%s"`, attribute.Key, html.EscapeString(value)))
+	}
+
+	extraAttrNames, extraHTMLAttributes := getExtraAttributes(tagName)
+	if len(extraAttrNames) > 0 {
+		attrNames = append(attrNames, extraAttrNames...)
+		htmlAttrs = append(htmlAttrs, extraHTMLAttributes...)
+	}
+
+	return attrNames, strings.Join(htmlAttrs, " ")
+}
+
+func getExtraAttributes(tagName string) ([]string, []string) {
+	switch tagName {
+	case "a":
+		return []string{"rel", "target", "referrerpolicy"}, []string{`rel="noopener noreferrer"`, `target="_blank"`, `referrerpolicy="no-referrer"`}
+	case "video", "audio":
+		return []string{"controls"}, []string{"controls"}
+	case "iframe":
+		return []string{"sandbox", "loading"}, []string{`sandbox="allow-scripts allow-same-origin allow-popups"`, `loading="lazy"`}
+	case "img":
+		return []string{"loading"}, []string{`loading="lazy"`}
+	default:
+		return nil, nil
+	}
+}
+
+func isValidTag(tagName string) bool {
+	for element := range getTagAllowList() {
+		if tagName == element {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isValidAttribute(tagName, attributeName string) bool {
+	for element, attributes := range getTagAllowList() {
+		if tagName == element {
+			if inList(attributeName, attributes) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+func isExternalResourceAttribute(attribute string) bool {
+	switch attribute {
+	case "src", "href", "poster", "cite":
+		return true
+	default:
+		return false
+	}
+}
+
+func hasRequiredAttributes(tagName string, attributes []string) bool {
+	elements := make(map[string][]string)
+	elements["a"] = []string{"href"}
+	elements["iframe"] = []string{"src"}
+	elements["img"] = []string{"src"}
+	elements["source"] = []string{"src", "srcset"}
+
+	for element, attrs := range elements {
+		if tagName == element {
+			for _, attribute := range attributes {
+				for _, attr := range attrs {
+					if attr == attribute {
+						return true
+					}
+				}
+			}
+
+			return false
+		}
+	}
+
+	return true
+}
+
+// See https://www.iana.org/assignments/uri-schemes/uri-schemes.xhtml
+func hasValidURIScheme(src string) bool {
+	whitelist := []string{
+		"apt:",
+		"bitcoin:",
+		"callto:",
+		"dav:",
+		"davs:",
+		"ed2k://",
+		"facetime://",
+		"feed:",
+		"ftp://",
+		"geo:",
+		"gopher://",
+		"git://",
+		"http://",
+		"https://",
+		"irc://",
+		"irc6://",
+		"ircs://",
+		"itms://",
+		"itms-apps://",
+		"magnet:",
+		"mailto:",
+		"news:",
+		"nntp:",
+		"rtmp://",
+		"sip:",
+		"sips:",
+		"skype:",
+		"spotify:",
+		"ssh://",
+		"sftp://",
+		"steam://",
+		"svn://",
+		"svn+ssh://",
+		"tel:",
+		"webcal://",
+		"xmpp:",
+	}
+
+	for _, prefix := range whitelist {
+		if strings.HasPrefix(src, prefix) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isBlockedResource(src string) bool {
+	blacklist := []string{
+		"feedsportal.com",
+		"api.flattr.com",
+		"stats.wordpress.com",
+		"plus.google.com/share",
+		"twitter.com/share",
+		"feeds.feedburner.com",
+	}
+
+	for _, element := range blacklist {
+		if strings.Contains(src, element) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isValidIframeSource(baseURL, src string) bool {
+	whitelist := []string{
+		"bandcamp.com",
+		"cdn.embedly.com",
+		"invidio.us",
+		"player.bilibili.com",
+		"player.vimeo.com",
+		"soundcloud.com",
+		"vk.com",
+		"w.soundcloud.com",
+		"www.dailymotion.com",
+		"www.youtube-nocookie.com",
+		"www.youtube.com",
+	}
+
+	domain := htmlutil.URLDomain(src)
+	// allow iframe from same origin
+	if htmlutil.URLDomain(baseURL) == domain {
+		return true
+	}
+
+	for _, safeDomain := range whitelist {
+		if safeDomain == domain {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getTagAllowList() map[string][]string {
+	whitelist := make(map[string][]string)
+	whitelist["img"] = []string{"alt", "title", "src", "srcset", "sizes"}
+	whitelist["picture"] = []string{}
+	whitelist["audio"] = []string{"src"}
+	whitelist["video"] = []string{"poster", "height", "width", "src"}
+	whitelist["source"] = []string{"src", "type", "srcset", "sizes", "media"}
+	whitelist["dt"] = []string{}
+	whitelist["dd"] = []string{}
+	whitelist["dl"] = []string{}
+	whitelist["table"] = []string{}
+	whitelist["caption"] = []string{}
+	whitelist["thead"] = []string{}
+	whitelist["tfooter"] = []string{}
+	whitelist["tr"] = []string{}
+	whitelist["td"] = []string{"rowspan", "colspan"}
+	whitelist["th"] = []string{"rowspan", "colspan"}
+	whitelist["h1"] = []string{}
+	whitelist["h2"] = []string{}
+	whitelist["h3"] = []string{}
+	whitelist["h4"] = []string{}
+	whitelist["h5"] = []string{}
+	whitelist["h6"] = []string{}
+	whitelist["strong"] = []string{}
+	whitelist["em"] = []string{}
+	whitelist["code"] = []string{}
+	whitelist["pre"] = []string{}
+	whitelist["blockquote"] = []string{}
+	whitelist["q"] = []string{"cite"}
+	whitelist["p"] = []string{}
+	whitelist["ul"] = []string{}
+	whitelist["li"] = []string{}
+	whitelist["ol"] = []string{}
+	whitelist["br"] = []string{}
+	whitelist["del"] = []string{}
+	whitelist["a"] = []string{"href", "title"}
+	whitelist["figure"] = []string{}
+	whitelist["figcaption"] = []string{}
+	whitelist["cite"] = []string{}
+	whitelist["time"] = []string{"datetime"}
+	whitelist["abbr"] = []string{"title"}
+	whitelist["acronym"] = []string{"title"}
+	whitelist["wbr"] = []string{}
+	whitelist["dfn"] = []string{}
+	whitelist["sub"] = []string{}
+	whitelist["sup"] = []string{}
+	whitelist["var"] = []string{}
+	whitelist["samp"] = []string{}
+	whitelist["s"] = []string{}
+	whitelist["del"] = []string{}
+	whitelist["ins"] = []string{}
+	whitelist["kbd"] = []string{}
+	whitelist["rp"] = []string{}
+	whitelist["rt"] = []string{}
+	whitelist["rtc"] = []string{}
+	whitelist["ruby"] = []string{}
+	whitelist["iframe"] = []string{"width", "height", "frameborder", "src", "allowfullscreen"}
+	return whitelist
+}
+
+func inList(needle string, haystack []string) bool {
+	for _, element := range haystack {
+		if element == needle {
+			return true
+		}
+	}
+
+	return false
+}
+
+func isBlockedTag(tagName string) bool {
+	blacklist := []string{
+		"noscript",
+		"script",
+		"style",
+	}
+
+	for _, element := range blacklist {
+		if element == tagName {
+			return true
+		}
+	}
+
+	return false
+}
+
+/*
+
+One or more strings separated by commas, indicating possible image sources for the user agent to use.
+
+Each string is composed of:
+- A URL to an image
+- Optionally, whitespace followed by one of:
+- A width descriptor (a positive integer directly followed by w). The width descriptor is divided by the source size given in the sizes attribute to calculate the effective pixel density.
+- A pixel density descriptor (a positive floating point number directly followed by x).
+
+*/
+func sanitizeSrcsetAttr(baseURL, value string) string {
+	var sanitizedSources []string
+	rawSources := splitSrcsetRegex.Split(value, -1)
+	for _, rawSource := range rawSources {
+		parts := strings.Split(strings.TrimSpace(rawSource), " ")
+		nbParts := len(parts)
+
+		if nbParts > 0 {
+			sanitizedSource := parts[0]
+			if !strings.HasPrefix(parts[0], "data:") {
+				sanitizedSource = htmlutil.AbsoluteUrl(parts[0], baseURL)
+				if sanitizedSource == "" {
+					continue
+				}
+			}
+
+			if nbParts == 2 && isValidWidthOrDensityDescriptor(parts[1]) {
+				sanitizedSource += " " + parts[1]
+			}
+
+			sanitizedSources = append(sanitizedSources, sanitizedSource)
+		}
+	}
+	return strings.Join(sanitizedSources, ", ")
+}
+
+func isValidWidthOrDensityDescriptor(value string) bool {
+	if value == "" {
+		return false
+	}
+
+	lastChar := value[len(value)-1:]
+	if lastChar != "w" && lastChar != "x" {
+		return false
+	}
+
+	_, err := strconv.ParseFloat(value[0:len(value)-1], 32)
+	return err == nil
+}
+
+func isValidDataAttribute(value string) bool {
+	var dataAttributeAllowList = []string{
+		"data:image/avif",
+		"data:image/apng",
+		"data:image/png",
+		"data:image/svg",
+		"data:image/svg+xml",
+		"data:image/jpg",
+		"data:image/jpeg",
+		"data:image/gif",
+		"data:image/webp",
+	}
+
+	for _, prefix := range dataAttributeAllowList {
+		if strings.HasPrefix(value, prefix) {
+			return true
+		}
+	}
+	return false
+}
--- a/src/content/sanitizer/sanitizer_test.go
+++ b/src/content/sanitizer/sanitizer_test.go
@@ -0,0 +1,492 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package scraper
+
+import "testing"
+
+func TestValidInput(t *testing.T) {
+	input := `<p>This is a <strong>text</strong> with an image: <img src="http://example.org/" alt="Test" loading="lazy">.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if input != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, input, output)
+	}
+}
+
+func TestImgWithTextDataURL(t *testing.T) {
+	input := `<img src="data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==" alt="Example">`
+	expected := ``
+	output := Sanitize("http://example.org/", input)
+
+	if output != expected {
+		t.Errorf(`Wrong output: %s`, output)
+	}
+}
+
+func TestImgWithDataURL(t *testing.T) {
+	input := `<img src="data:image/gif;base64,test" alt="Example">`
+	expected := `<img src="data:image/gif;base64,test" alt="Example" loading="lazy">`
+	output := Sanitize("http://example.org/", input)
+
+	if output != expected {
+		t.Errorf(`Wrong output: %s`, output)
+	}
+}
+
+func TestImgWithSrcset(t *testing.T) {
+	input := `<img srcset="example-320w.jpg, example-480w.jpg 1.5x,   example-640w.jpg 2x, example-640w.jpg 640w" src="example-640w.jpg" alt="Example">`
+	expected := `<img srcset="http://example.org/example-320w.jpg, http://example.org/example-480w.jpg 1.5x, http://example.org/example-640w.jpg 2x, http://example.org/example-640w.jpg 640w" src="http://example.org/example-640w.jpg" alt="Example" loading="lazy">`
+	output := Sanitize("http://example.org/", input)
+
+	if output != expected {
+		t.Errorf(`Wrong output: %s`, output)
+	}
+}
+
+func TestImgWithSrcsetAndDataURL(t *testing.T) {
+	input := `<img srcset="data:image/gif;base64,test" src="http://example.org/example-320w.jpg" alt="Example">`
+	expected := `<img srcset="data:image/gif;base64,test" src="http://example.org/example-320w.jpg" alt="Example" loading="lazy">`
+	output := Sanitize("http://example.org/", input)
+
+	if output != expected {
+		t.Errorf(`Wrong output: %s`, output)
+	}
+}
+
+func TestSourceWithSrcsetAndMedia(t *testing.T) {
+	input := `<picture><source media="(min-width: 800px)" srcset="elva-800w.jpg"></picture>`
+	expected := `<picture><source media="(min-width: 800px)" srcset="http://example.org/elva-800w.jpg"></picture>`
+	output := Sanitize("http://example.org/", input)
+
+	if output != expected {
+		t.Errorf(`Wrong output: %s`, output)
+	}
+}
+
+func TestMediumImgWithSrcset(t *testing.T) {
+	input := `<img alt="Image for post" class="t u v ef aj" src="https://miro.medium.com/max/5460/1*aJ9JibWDqO81qMfNtqgqrw.jpeg" srcset="https://miro.medium.com/max/552/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 276w, https://miro.medium.com/max/1000/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 500w" sizes="500px" width="2730" height="3407">`
+	expected := `<img alt="Image for post" src="https://miro.medium.com/max/5460/1*aJ9JibWDqO81qMfNtqgqrw.jpeg" srcset="https://miro.medium.com/max/552/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 276w, https://miro.medium.com/max/1000/1*aJ9JibWDqO81qMfNtqgqrw.jpeg 500w" sizes="500px" loading="lazy">`
+	output := Sanitize("http://example.org/", input)
+
+	if output != expected {
+		t.Errorf(`Wrong output: %s`, output)
+	}
+}
+
+func TestSelfClosingTags(t *testing.T) {
+	input := `<p>This <br> is a <strong>text</strong> <br/>with an image: <img src="http://example.org/" alt="Test" loading="lazy"/>.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if input != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, input, output)
+	}
+}
+
+func TestTable(t *testing.T) {
+	input := `<table><tr><th>A</th><th colspan="2">B</th></tr><tr><td>C</td><td>D</td><td>E</td></tr></table>`
+	output := Sanitize("http://example.org/", input)
+
+	if input != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, input, output)
+	}
+}
+
+func TestRelativeURL(t *testing.T) {
+	input := `This <a href="/test.html">link is relative</a> and this image: <img src="../folder/image.png"/>`
+	expected := `This <a href="http://example.org/test.html" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">link is relative</a> and this image: <img src="http://example.org/folder/image.png" loading="lazy"/>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestProtocolRelativeURL(t *testing.T) {
+	input := `This <a href="//static.example.org/index.html">link is relative</a>.`
+	expected := `This <a href="http://static.example.org/index.html" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">link is relative</a>.`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestInvalidTag(t *testing.T) {
+	input := `<p>My invalid <b>tag</b>.</p>`
+	expected := `<p>My invalid tag.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestVideoTag(t *testing.T) {
+	input := `<p>My valid <video src="videofile.webm" autoplay poster="posterimage.jpg">fallback</video>.</p>`
+	expected := `<p>My valid <video src="http://example.org/videofile.webm" poster="http://example.org/posterimage.jpg" controls>fallback</video>.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestAudioAndSourceTag(t *testing.T) {
+	input := `<p>My music <audio controls="controls"><source src="foo.wav" type="audio/wav"></audio>.</p>`
+	expected := `<p>My music <audio controls><source src="http://example.org/foo.wav" type="audio/wav"></audio>.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestUnknownTag(t *testing.T) {
+	input := `<p>My invalid <unknown>tag</unknown>.</p>`
+	expected := `<p>My invalid tag.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestInvalidNestedTag(t *testing.T) {
+	input := `<p>My invalid <b>tag with some <em>valid</em> tag</b>.</p>`
+	expected := `<p>My invalid tag with some <em>valid</em> tag.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestInvalidIFrame(t *testing.T) {
+	input := `<iframe src="http://example.org/"></iframe>`
+	expected := ``
+	output := Sanitize("http://example.com/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestIFrameWithChildElements(t *testing.T) {
+	input := `<iframe src="https://www.youtube.com/"><p>test</p></iframe>`
+	expected := `<iframe src="https://www.youtube.com/" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	output := Sanitize("http://example.com/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestInvalidURLScheme(t *testing.T) {
+	input := `<p>This link is <a src="file:///etc/passwd">not valid</a></p>`
+	expected := `<p>This link is not valid</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestAPTURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="apt:some-package?channel=test">valid</a></p>`
+	expected := `<p>This link is <a href="apt:some-package?channel=test" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestBitcoinURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="bitcoin:175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W">valid</a></p>`
+	expected := `<p>This link is <a href="bitcoin:175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestCallToURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="callto:12345679">valid</a></p>`
+	expected := `<p>This link is <a href="callto:12345679" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestFeedURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="feed://example.com/rss.xml">valid</a></p>`
+	expected := `<p>This link is <a href="feed://example.com/rss.xml" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+
+	input = `<p>This link is <a href="feed:https://example.com/rss.xml">valid</a></p>`
+	expected = `<p>This link is <a href="feed:https://example.com/rss.xml" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output = Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestGeoURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="geo:13.4125,103.8667">valid</a></p>`
+	expected := `<p>This link is <a href="geo:13.4125,103.8667" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestItunesURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="itms://itunes.com/apps/my-app-name">valid</a></p>`
+	expected := `<p>This link is <a href="itms://itunes.com/apps/my-app-name" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+
+	input = `<p>This link is <a href="itms-apps://itunes.com/apps/my-app-name">valid</a></p>`
+	expected = `<p>This link is <a href="itms-apps://itunes.com/apps/my-app-name" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output = Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestMagnetURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="magnet:?xt.1=urn:sha1:YNCKHTQCWBTRNJIV4WNAE52SJUQCZO5C&amp;xt.2=urn:sha1:TXGCZQTH26NL6OUQAJJPFALHG2LTGBC7">valid</a></p>`
+	expected := `<p>This link is <a href="magnet:?xt.1=urn:sha1:YNCKHTQCWBTRNJIV4WNAE52SJUQCZO5C&amp;xt.2=urn:sha1:TXGCZQTH26NL6OUQAJJPFALHG2LTGBC7" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestMailtoURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="mailto:jsmith@example.com?subject=A%20Test&amp;body=My%20idea%20is%3A%20%0A">valid</a></p>`
+	expected := `<p>This link is <a href="mailto:jsmith@example.com?subject=A%20Test&amp;body=My%20idea%20is%3A%20%0A" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestNewsURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="news://news.server.example/*">valid</a></p>`
+	expected := `<p>This link is <a href="news://news.server.example/*" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+
+	input = `<p>This link is <a href="news:example.group.this">valid</a></p>`
+	expected = `<p>This link is <a href="news:example.group.this" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output = Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+
+	input = `<p>This link is <a href="nntp://news.server.example/example.group.this">valid</a></p>`
+	expected = `<p>This link is <a href="nntp://news.server.example/example.group.this" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output = Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestRTMPURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="rtmp://mycompany.com/vod/mp4:mycoolvideo.mov">valid</a></p>`
+	expected := `<p>This link is <a href="rtmp://mycompany.com/vod/mp4:mycoolvideo.mov" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestSIPURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="sip:+1-212-555-1212:1234@gateway.com;user=phone">valid</a></p>`
+	expected := `<p>This link is <a href="sip:+1-212-555-1212:1234@gateway.com;user=phone" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+
+	input = `<p>This link is <a href="sips:alice@atlanta.com?subject=project%20x&amp;priority=urgent">valid</a></p>`
+	expected = `<p>This link is <a href="sips:alice@atlanta.com?subject=project%20x&amp;priority=urgent" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output = Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestSkypeURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="skype:echo123?call">valid</a></p>`
+	expected := `<p>This link is <a href="skype:echo123?call" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestSpotifyURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="spotify:track:2jCnn1QPQ3E8ExtLe6INsx">valid</a></p>`
+	expected := `<p>This link is <a href="spotify:track:2jCnn1QPQ3E8ExtLe6INsx" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestSteamURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="steam://settings/account">valid</a></p>`
+	expected := `<p>This link is <a href="steam://settings/account" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestSubversionURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="svn://example.org">valid</a></p>`
+	expected := `<p>This link is <a href="svn://example.org" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+
+	input = `<p>This link is <a href="svn+ssh://example.org">valid</a></p>`
+	expected = `<p>This link is <a href="svn+ssh://example.org" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output = Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestTelURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="tel:+1-201-555-0123">valid</a></p>`
+	expected := `<p>This link is <a href="tel:+1-201-555-0123" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestWebcalURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="webcal://example.com/calendar.ics">valid</a></p>`
+	expected := `<p>This link is <a href="webcal://example.com/calendar.ics" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestXMPPURIScheme(t *testing.T) {
+	input := `<p>This link is <a href="xmpp:user@host?subscribe&amp;type=subscribed">valid</a></p>`
+	expected := `<p>This link is <a href="xmpp:user@host?subscribe&amp;type=subscribed" rel="noopener noreferrer" target="_blank" referrerpolicy="no-referrer">valid</a></p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestBlacklistedLink(t *testing.T) {
+	input := `<p>This image is not valid <img src="https://stats.wordpress.com/some-tracker"></p>`
+	expected := `<p>This image is not valid </p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestXmlEntities(t *testing.T) {
+	input := `<pre>echo "test" &gt; /etc/hosts</pre>`
+	expected := `<pre>echo &#34;test&#34; &gt; /etc/hosts</pre>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestEspaceAttributes(t *testing.T) {
+	input := `<td rowspan="<b>test</b>">test</td>`
+	expected := `<td rowspan="&lt;b&gt;test&lt;/b&gt;">test</td>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestReplaceIframeURL(t *testing.T) {
+	input := `<iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0"></iframe>`
+	expected := `<iframe src="https://player.vimeo.com/video/123456?title=0&amp;byline=0" sandbox="allow-scripts allow-same-origin allow-popups" loading="lazy"></iframe>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestReplaceNoScript(t *testing.T) {
+	input := `<p>Before paragraph.</p><noscript>Inside <code>noscript</code> tag with an image: <img src="http://example.org/" alt="Test" loading="lazy"></noscript><p>After paragraph.</p>`
+	expected := `<p>Before paragraph.</p><p>After paragraph.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestReplaceScript(t *testing.T) {
+	input := `<p>Before paragraph.</p><script type="text/javascript">alert("1");</script><p>After paragraph.</p>`
+	expected := `<p>Before paragraph.</p><p>After paragraph.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
+
+func TestReplaceStyle(t *testing.T) {
+	input := `<p>Before paragraph.</p><style>body { background-color: #ff0000; }</style><p>After paragraph.</p>`
+	expected := `<p>Before paragraph.</p><p>After paragraph.</p>`
+	output := Sanitize("http://example.org/", input)
+
+	if expected != output {
+		t.Errorf(`Wrong output: "%s" != "%s"`, expected, output)
+	}
+}
--- a/src/content/scraper/finder.go
+++ b/src/content/scraper/finder.go
@@ -0,0 +1,97 @@
+package scraper
+
+import (
+	"strings"
+
+	"github.com/nkanaev/yarr/src/content/htmlutil"
+	"golang.org/x/net/html"
+)
+
+func FindFeeds(body string, base string) map[string]string {
+	candidates := make(map[string]string)
+
+	doc, err := html.Parse(strings.NewReader(body))
+	if err != nil {
+		return candidates
+	}
+
+	// find direct links
+	// css: link[type=application/atom+xml]
+	linkTypes := []string{"application/atom+xml", "application/rss+xml", "application/json"}
+	isFeedLink := func(n *html.Node) bool {
+		if n.Type == html.ElementNode && n.Data == "link" {
+			t := htmlutil.Attr(n, "type")
+			for _, tt := range linkTypes {
+				if tt == t {
+					return true
+				}
+			}
+		}
+		return false
+	}
+	for _, node := range htmlutil.FindNodes(doc, isFeedLink) {
+		href := htmlutil.Attr(node, "href")
+		name := htmlutil.Attr(node, "title")
+		link := htmlutil.AbsoluteUrl(href, base)
+		if link != "" {
+			candidates[link] = name
+		}
+	}
+
+	// guess by hyperlink properties
+	if len(candidates) == 0 {
+		// css: a[href="feed"]
+		// css: a:contains("rss")
+		feedHrefs := []string{"feed", "feed.xml", "rss.xml", "atom.xml"}
+		feedTexts := []string{"rss", "feed"}
+		isFeedHyperLink := func(n *html.Node) bool {
+			if n.Type == html.ElementNode && n.Data == "a" {
+				href := strings.Trim(htmlutil.Attr(n, "href"), "/")
+				for _, feedHref := range feedHrefs {
+					if strings.HasSuffix(href, feedHref) {
+						return true
+					}
+				}
+				text := htmlutil.Text(n)
+				for _, feedText := range feedTexts {
+					if strings.EqualFold(text, feedText) {
+						return true
+					}
+				}
+			}
+			return false
+		}
+		for _, node := range htmlutil.FindNodes(doc, isFeedHyperLink) {
+			href := htmlutil.Attr(node, "href")
+			link := htmlutil.AbsoluteUrl(href, base)
+			if link != "" {
+				candidates[link] = ""
+			}
+		}
+	}
+
+	return candidates
+}
+
+func FindIcons(body string, base string) []string {
+	icons := make([]string, 0)
+
+	doc, err := html.Parse(strings.NewReader(body))
+	if err != nil {
+		return icons
+	}
+
+	// css: link[rel=icon]
+	isLink := func(n *html.Node) bool {
+		return n.Type == html.ElementNode && n.Data == "link"
+	}
+	for _, node := range htmlutil.FindNodes(doc, isLink) {
+		rels := strings.Split(htmlutil.Attr(node, "rel"), " ")
+		for _, rel := range rels {
+			if strings.EqualFold(rel, "icon") {
+				icons = append(icons, htmlutil.AbsoluteUrl(htmlutil.Attr(node, "href"), base))
+			}
+		}
+	}
+	return icons
+}
--- a/src/content/scraper/finder_test.go
+++ b/src/content/scraper/finder_test.go
@@ -0,0 +1,97 @@
+package scraper
+
+import (
+	"reflect"
+	"testing"
+)
+
+const base = "http://example.com"
+
+func TestFindFeedsInvalidHTML(t *testing.T) {
+	x := `some nonsense`
+	r := FindFeeds(x, base)
+	if len(r) != 0 {
+		t.Fatal("not expecting results")
+	}
+}
+
+func TestFindFeedsLinks(t *testing.T) {
+	x := `
+		<!DOCTYPE html>
+		<html lang="en">
+		<head>
+			<meta charset="UTF-8">
+			<title></title>
+			<link rel="alternate" href="/feed.xml" type="application/rss+xml" title="rss with title">
+			<link rel="alternate" href="/atom.xml" type="application/atom+xml">
+			<link rel="alternate" href="/feed.json" type="application/json">
+		</head>
+		<body>
+			<a href="/feed.xml">rss</a>
+		</body>
+		</html>
+	`
+	have := FindFeeds(x, base)
+
+	want := map[string]string{
+		base + "/feed.xml":  "rss with title",
+		base + "/atom.xml":  "",
+		base + "/feed.json": "",
+	}
+	if !reflect.DeepEqual(have, want) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid result")
+	}
+}
+
+func TestFindFeedsGuess(t *testing.T) {
+	body := `
+		<!DOCTYPE html>
+		<html lang="en">
+		<body>
+			<!-- negative -->
+			<a href="/about">what is rss?</a>
+			<a href="/feed/cows">moo</a>
+
+			<!-- positive -->
+			<a href="/feed.xml">subscribe</a>
+			<a href="/news">rss</a>
+		</body>
+		</html>
+	`
+	have := FindFeeds(body, base)
+	want := map[string]string{
+		base + "/feed.xml": "",
+		base + "/news":     "",
+	}
+	if !reflect.DeepEqual(want, have) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid result")
+	}
+}
+
+func TestFindIcons(t *testing.T) {
+	body := `
+		<!DOCTYPE html>
+		<html lang="en">
+		<head>
+			<meta charset="UTF-8">
+			<title></title>
+			<link rel="icon favicon" href="/favicon.ico">
+			<link rel="icon macicon" href="path/to/favicon.png">
+		</head>
+		<body>
+			
+		</body>
+		</html>
+	`
+	have := FindIcons(body, base)
+	want := []string{base + "/favicon.ico", base + "/path/to/favicon.png"}
+	if !reflect.DeepEqual(have, want) {
+		t.Logf("want: %#v", want)
+		t.Logf("have: %#v", have)
+		t.Fatal("invalid result")
+	}
+}