james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2026-06-27 18:45:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

add storage interface, fix all references

Browse Source
This commit is contained in:
nkanaev
2026-06-10 22:24:04 +01:00
parent 3f10371975
commit f2556178b3
9 changed files with 65 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/worker/crawler.go
View File

@@ -14,6 +14,7 @@ import (
"github.com/nkanaev/yarr/src/content/scraper"
"github.com/nkanaev/yarr/src/parser"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/storage/model"
"golang.org/x/net/html/charset"
)
@@ -139,28 +140,28 @@ func findFavicon(siteUrl, feedUrl string) (*[]byte, error) {
return &emptyIcon, nil
}
func ConvertItems(items []parser.Item, feed storage.Feed) []storage.Item {
result := make([]storage.Item, len(items))
func ConvertItems(items []parser.Item, feed model.Feed) []model.Item {
result := make([]model.Item, len(items))
for i, item := range items {
mediaLinks := make(storage.MediaLinks, 0)
mediaLinks := make(model.MediaLinks, 0)
for _, link := range item.MediaLinks {
mediaLinks = append(mediaLinks, storage.MediaLink(link))
mediaLinks = append(mediaLinks, model.MediaLink(link))
}
result[i] = storage.Item{
result[i] = model.Item{
GUID: item.GUID,
FeedId: feed.Id,
Title: item.Title,
Link: item.URL,
Content: item.Content,
Date: item.Date,
Status: storage.UNREAD,
Status: model.UNREAD,
MediaLinks: mediaLinks,
}
}
return result
}
func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
func listItems(f model.Feed, db storage.Storage) ([]model.Item, error) {
lmod := ""
etag := ""
if state, _ := db.GetFeedState(f.Id); state != nil {
@@ -193,7 +194,7 @@ func listItems(f storage.Feed, db *storage.Storage) ([]storage.Item, error) {
etag = res.Header.Get("Etag")
now := time.Now().UTC()
if lmod != "" || etag != "" {
db.UpdateFeedState(f.Id, storage.UpdateFeedStateParams{
db.UpdateFeedState(f.Id, model.UpdateFeedStateParams{
HTTPLastModified: &lmod,
HTTPEtag: &etag,
LastRefreshed: &now,

21
src/worker/worker.go
View File

@@ -7,19 +7,20 @@ import (
"time"
"github.com/nkanaev/yarr/src/storage"
"github.com/nkanaev/yarr/src/storage/model"
)
const NUM_WORKERS = 4
type Worker struct {
db *storage.Storage
db storage.Storage
pending *int32
refresh *time.Ticker
reflock sync.Mutex
stopper chan bool
}
func NewWorker(db *storage.Storage) *Worker {
func NewWorker(db storage.Storage) *Worker {
pending := int32(0)
return &Worker{db: db, pending: &pending}
}
@@ -39,13 +40,13 @@ func (w *Worker) StartFeedCleaner() {
}()
}
func (w *Worker) FindFeedFavicon(feed storage.Feed) {
func (w *Worker) FindFeedFavicon(feed model.Feed) {
icon, err := findFavicon(feed.Link, feed.FeedLink)
if err != nil {
log.Printf("Failed to find favicon for %s (%s): %s", feed.FeedLink, feed.Link, err)
}
if icon != nil {
w.db.UpdateFeed(feed.Id, storage.UpdateFeedParams{Icon: storage.SetNullable(icon)})
w.db.UpdateFeed(feed.Id, model.UpdateFeedParams{Icon: model.SetNullable(icon)})
}
}
@@ -99,11 +100,11 @@ func (w *Worker) RefreshFeeds() {
go w.refresher(feeds)
}
func (w *Worker) refresher(feeds []storage.Feed) {
func (w *Worker) refresher(feeds []model.Feed) {
// w.db.ResetFeedErrors()
srcqueue := make(chan storage.Feed, len(feeds))
dstqueue := make(chan []storage.Item)
srcqueue := make(chan model.Feed, len(feeds))
dstqueue := make(chan []model.Item)
for range NUM_WORKERS {
go w.worker(srcqueue, dstqueue)
@@ -125,15 +126,15 @@ func (w *Worker) refresher(feeds []storage.Feed) {
log.Printf("Finished refreshing %d feeds", len(feeds))
}
func (w *Worker) worker(srcqueue <-chan storage.Feed, dstqueue chan<- []storage.Item) {
func (w *Worker) worker(srcqueue <-chan model.Feed, dstqueue chan<- []model.Item) {
for feed := range srcqueue {
empty := ""
w.db.UpdateFeedState(feed.Id, storage.UpdateFeedStateParams{LastError: &empty})
w.db.UpdateFeedState(feed.Id, model.UpdateFeedStateParams{LastError: &empty})
items, err := listItems(feed, w.db)
if err != nil {
errMsg := err.Error()
w.db.UpdateFeedState(feed.Id, storage.UpdateFeedStateParams{LastError: &errMsg})
w.db.UpdateFeedState(feed.Id, model.UpdateFeedStateParams{LastError: &errMsg})
}
if len(items) > 0 && !feed.HasIcon {
w.FindFeedFavicon(feed)