james/yarr
4
0
Fork 0
mirror of https://github.com/nkanaev/yarr.git synced 2025-05-25 21:49:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge ee34959d7c7c5a7e689026f1825e14e661e5a034 into 6d16e93008ba8af66c40f335f7bdcfdd2e436cd2

Browse Source
This commit is contained in:
icefed 2023-08-20 14:41:36 +08:00 committed by GitHub
commit 93097f3261
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 456 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/server/auth/middleware.go
View File

@ -12,7 +12,7 @@ type Middleware struct {
Username string
Password string
BasePath string
Public string
Public []string
}
func unsafeMethod(method string) bool {
@ -20,10 +20,12 @@ func unsafeMethod(method string) bool {
}
func (m *Middleware) Handler(c *router.Context) {
if strings.HasPrefix(c.Req.URL.Path, m.BasePath+m.Public) {
for _, path := range m.Public {
if strings.HasPrefix(c.Req.URL.Path, m.BasePath+path) {
c.Next()
return
}
}
if IsAuthenticated(c.Req, m.Username, m.Password) {
c.Next()
return

385
src/server/fever.go Normal file
View File

@ -0,0 +1,385 @@
package server
import (
"crypto/md5"
"encoding/base64"
"fmt"
"log"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/nkanaev/yarr/src/server/auth"
"github.com/nkanaev/yarr/src/server/router"
"github.com/nkanaev/yarr/src/storage"
)
type FeverGroup struct {
ID int64 `json:"id"`
Title string `json:"title"`
}
type FeverFeedsGroup struct {
GroupID int64 `json:"group_id"`
FeedIDs string `json:"feed_ids"`
}
type FeverFeed struct {
ID int64 `json:"id"`
FaviconID int64 `json:"favicon_id"`
Title string `json:"title"`
Url string `json:"url"`
SiteUrl string `json:"site_url"`
IsSpark int `json:"is_spark"`
LastUpdated int64 `json:"last_updated_on_time"`
}
type FeverItem struct {
ID int64 `json:"id"`
FeedID int64 `json:"feed_id"`
Title string `json:"title"`
Author string `json:"author"`
HTML string `json:"html"`
Url string `json:"url"`
IsSaved int `json:"is_saved"`
IsRead int `json:"is_read"`
CreatedAt int64 `json:"created_on_time"`
}
type FeverFavicon struct {
ID int64 `json:"id"`
Data string `json:"data"`
}
func writeFeverJSON(c *router.Context, data map[string]interface{}, lastRefreshed int64) {
data["api_version"] = 1
data["auth"] = 1
data["last_refreshed_on_time"] = lastRefreshed
c.JSON(http.StatusOK, data)
}
func getLastRefreshedOnTime(httpStates map[int64]storage.HTTPState) int64 {
if len(httpStates) == 0 {
return 0
}
var lastRefreshed int64
for _, state := range httpStates {
if state.LastRefreshed.Unix() > lastRefreshed {
lastRefreshed = state.LastRefreshed.Unix()
}
}
return lastRefreshed
}
func (s *Server) feverAuth(c *router.Context) bool {
if s.Username != "" && s.Password != "" {
apiKey := c.Req.FormValue("api_key")
md5HashValue := md5.Sum([]byte(fmt.Sprintf("%s:%s", s.Username, s.Password)))
hexMD5HashValue := fmt.Sprintf("%x", md5HashValue[:])
if auth.StringsEqual(apiKey, hexMD5HashValue) {
return false
}
}
return true
}
func formHasValue(values url.Values, value string) bool {
if _, ok := values[value]; ok {
return true
}
return false
}
func (s *Server) handleFever(c *router.Context) {
c.Req.ParseForm()
if !s.feverAuth(c) {
c.JSON(http.StatusOK, map[string]interface{}{
"api_version": 1,
"auth": 0,
"last_refreshed_on_time": 0,
})
return
}
switch {
case formHasValue(c.Req.Form, "groups"):
s.feverGroupsHandler(c)
case formHasValue(c.Req.Form, "feeds"):
s.feverFeedsHandler(c)
case formHasValue(c.Req.Form, "unread_item_ids"):
s.feverUnreadItemIDsHandler(c)
case formHasValue(c.Req.Form, "saved_item_ids"):
s.feverSavedItemIDsHandler(c)
case formHasValue(c.Req.Form, "favicons"):
s.feverFaviconsHandler(c)
case formHasValue(c.Req.Form, "items"):
s.feverItemsHandler(c)
case formHasValue(c.Req.Form, "links"):
s.feverLinksHandler(c)
case formHasValue(c.Req.Form, "mark"):
s.feverMarkHandler(c)
default:
c.JSON(http.StatusOK, map[string]interface{}{
"api_version": 1,
"auth": 1,
"last_refreshed_on_time": getLastRefreshedOnTime(s.db.ListHTTPStates()),
})
}
}
func joinInts(values []int64) string {
var result strings.Builder
for i, val := range values {
fmt.Fprintf(&result, "%d", val)
if i != len(values)-1 {
result.WriteString(",")
}
}
return result.String()
}
func feedGroups(db *storage.Storage) []*FeverFeedsGroup {
feeds := db.ListFeeds()
groupFeeds := make(map[int64][]int64)
for _, feed := range feeds {
if feed.FolderId == nil {
continue
}
groupFeeds[*feed.FolderId] = append(groupFeeds[*feed.FolderId], feed.Id)
}
result := make([]*FeverFeedsGroup, 0)
for groupId, feedIds := range groupFeeds {
result = append(result, &FeverFeedsGroup{
GroupID: groupId,
FeedIDs: joinInts(feedIds),
})
}
return result
}
func (s *Server) feverGroupsHandler(c *router.Context) {
folders := s.db.ListFolders()
groups := make([]*FeverGroup, len(folders))
for i, folder := range folders {
groups[i] = &FeverGroup{ID: folder.Id, Title: folder.Title}
}
writeFeverJSON(c, map[string]interface{}{
"groups": groups,
"feeds_groups": feedGroups(s.db),
}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
}
func (s *Server) feverFeedsHandler(c *router.Context) {
feeds := s.db.ListFeeds()
httpStates := s.db.ListHTTPStates()
feverFeeds := make([]*FeverFeed, len(feeds))
for i, feed := range feeds {
var lastUpdated int64
if state, ok := httpStates[feed.Id]; ok {
lastUpdated = state.LastRefreshed.Unix()
}
feverFeeds[i] = &FeverFeed{
ID: feed.Id,
FaviconID: feed.Id,
Title: feed.Title,
Url: feed.FeedLink,
SiteUrl: feed.Link,
IsSpark: 0,
LastUpdated: lastUpdated,
}
}
writeFeverJSON(c, map[string]interface{}{
"feeds": feverFeeds,
"feeds_groups": feedGroups(s.db),
}, getLastRefreshedOnTime(httpStates))
}
func (s *Server) feverFaviconsHandler(c *router.Context) {
feeds := s.db.ListFeeds()
favicons := make([]*FeverFavicon, len(feeds))
for i, feed := range feeds {
data := "data:image/gif;base64,R0lGODlhAQABAAAAACw="
if feed.HasIcon {
icon := s.db.GetFeed(feed.Id).Icon
data = fmt.Sprintf(
"data:%s;base64,%s",
http.DetectContentType(*icon),
base64.StdEncoding.EncodeToString(*icon),
)
}
favicons[i] = &FeverFavicon{ID: feed.Id, Data: data}
}
writeFeverJSON(c, map[string]interface{}{
"favicons": favicons,
}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
}
// for memory pressure reasons, we only return a limited number of items
// documented at https://github.com/DigitalDJ/tinytinyrss-fever-plugin/blob/master/fever-api.md#items
const listLimit = 50
func (s *Server) feverItemsHandler(c *router.Context) {
filter := storage.ItemFilter{}
query := c.Req.URL.Query()
switch {
case query.Get("with_ids") != "":
ids := make([]int64, 0)
for _, idstr := range strings.Split(query.Get("with_ids"), ",") {
if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
ids = append(ids, idnum)
}
}
filter.IDs = &ids
case query.Get("since_id") != "":
idstr := query.Get("since_id")
if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
filter.SinceID = &idnum
}
case query.Get("max_id") != "":
idstr := query.Get("max_id")
if idnum, err := strconv.ParseInt(idstr, 10, 64); err == nil {
filter.MaxID = &idnum
}
}
items := s.db.ListItems(filter, listLimit, true, true)
feverItems := make([]FeverItem, len(items))
for i, item := range items {
date := item.Date
time := date.Unix()
isSaved := 0
if item.Status == storage.STARRED {
isSaved = 1
}
isRead := 0
if item.Status == storage.READ {
isRead = 1
}
feverItems[i] = FeverItem{
ID: item.Id,
FeedID: item.FeedId,
Title: item.Title,
Author: "",
HTML: item.Content,
Url: item.Link,
IsSaved: isSaved,
IsRead: isRead,
CreatedAt: time,
}
}
writeFeverJSON(c, map[string]interface{}{
"items": feverItems,
}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
}
func (s *Server) feverLinksHandler(c *router.Context) {
writeFeverJSON(c, map[string]interface{}{
"links": make([]interface{}, 0),
}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
}
func (s *Server) feverUnreadItemIDsHandler(c *router.Context) {
status := storage.UNREAD
itemIds := make([]int64, 0)
itemFilter := storage.ItemFilter{
Status: &status,
}
for {
items := s.db.ListItems(itemFilter, listLimit, true, false)
if len(items) == 0 {
break
}
for _, item := range items {
itemIds = append(itemIds, item.Id)
}
itemFilter.After = &items[len(items)-1].Id
}
writeFeverJSON(c, map[string]interface{}{
"unread_item_ids": joinInts(itemIds),
}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
}
func (s *Server) feverSavedItemIDsHandler(c *router.Context) {
status := storage.STARRED
itemIds := make([]int64, 0)
itemFilter := storage.ItemFilter{
Status: &status,
}
for {
items := s.db.ListItems(itemFilter, listLimit, true, false)
if len(items) == 0 {
break
}
for _, item := range items {
itemIds = append(itemIds, item.Id)
}
itemFilter.After = &items[len(items)-1].Id
}
writeFeverJSON(c, map[string]interface{}{
"saved_item_ids": joinInts(itemIds),
}, getLastRefreshedOnTime(s.db.ListHTTPStates()))
}
func (s *Server) feverMarkHandler(c *router.Context) {
id, err := strconv.ParseInt(c.Req.Form.Get("id"), 10, 64)
if err != nil {
log.Print("invalid id:", err)
return
}
switch c.Req.Form.Get("mark") {
case "item":
var status storage.ItemStatus
switch c.Req.Form.Get("as") {
case "read":
status = storage.READ
case "unread":
status = storage.UNREAD
case "saved":
status = storage.STARRED
case "unsaved":
status = storage.READ
default:
c.Out.WriteHeader(http.StatusBadRequest)
return
}
s.db.UpdateItemStatus(id, status)
case "feed":
if c.Req.Form.Get("as") != "read" {
c.Out.WriteHeader(http.StatusBadRequest)
}
markFilter := storage.MarkFilter{FeedID: &id}
x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
if x > 0 {
before := time.Unix(x, 0)
markFilter.Before = &before
}
s.db.MarkItemsRead(markFilter)
case "group":
if c.Req.Form.Get("as") != "read" {
c.Out.WriteHeader(http.StatusBadRequest)
}
markFilter := storage.MarkFilter{FolderID: &id}
x, _ := strconv.ParseInt(c.Req.Form.Get("before"), 10, 64)
if x > 0 {
before := time.Unix(x, 0)
markFilter.Before = &before
}
s.db.MarkItemsRead(markFilter)
default:
c.Out.WriteHeader(http.StatusBadRequest)
return
}
}

5
src/server/routes.go
View File

@ -33,7 +33,7 @@ func (s *Server) handler() http.Handler {
BasePath: s.BasePath,
Username: s.Username,
Password: s.Password,
Public: "/static",
Public: []string{"/static", "/fever"},
}
r.Use(a.Handler)
}
@ -56,6 +56,7 @@ func (s *Server) handler() http.Handler {
r.For("/opml/export", s.handleOPMLExport)
r.For("/page", s.handlePageCrawl)
r.For("/logout", s.handleLogout)
r.For("/fever/", s.handleFever)
return r
}
@ -355,7 +356,7 @@ func (s *Server) handleItemList(c *router.Context) {
}
newestFirst := query.Get("oldest_first") != "true"
items := s.db.ListItems(filter, perPage+1, newestFirst)
items := s.db.ListItems(filter, perPage+1, newestFirst, false)
hasMore := false
if len(items) == perPage+1 {
hasMore = true

54
src/storage/item.go
View File

@ -62,11 +62,17 @@ type ItemFilter struct {
Status *ItemStatus
Search *string
After *int64
IDs *[]int64
SinceID *int64
MaxID *int64
Before *time.Time
}
type MarkFilter struct {
FolderID *int64
FeedID *int64
Before *time.Time
}
func (s *Storage) CreateItems(items []Item) bool {
@ -140,6 +146,28 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
cond = append(cond, fmt.Sprintf("(i.date, i.id) %s (select date, id from items where id = ?)", compare))
args = append(args, *filter.After)
}
if filter.IDs != nil && len(*filter.IDs) > 0 {
qmarks := make([]string, len(*filter.IDs))
idargs := make([]interface{}, len(*filter.IDs))
for i, id := range *filter.IDs {
qmarks[i] = "?"
idargs[i] = id
}
cond = append(cond, "i.id in ("+strings.Join(qmarks, ",")+")")
args = append(args, idargs...)
}
if filter.SinceID != nil {
cond = append(cond, "i.id > ?")
args = append(args, filter.SinceID)
}
if filter.MaxID != nil {
cond = append(cond, "i.id < ?")
args = append(args, filter.MaxID)
}
if filter.Before != nil {
cond = append(cond, "i.date < ?")
args = append(args, filter.Before)
}
predicate := "1"
if len(cond) > 0 {
@ -149,7 +177,7 @@ func listQueryPredicate(filter ItemFilter, newestFirst bool) (string, []interfac
return predicate, args
}
func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []Item {
func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool, withContent bool) []Item {
predicate, args := listQueryPredicate(filter, newestFirst)
result := make([]Item, 0, 0)
@ -157,17 +185,23 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
if !newestFirst {
order = "date asc, id asc"
}
if filter.IDs != nil || filter.SinceID != nil || filter.MaxID != nil {
order = "i.id asc"
}
selectCols := "i.id, i.guid, i.feed_id, i.title, i.link, i.date, i.status, i.image, i.podcast_url"
if withContent {
selectCols += ", i.content"
} else {
selectCols += ", '' as content"
}
query := fmt.Sprintf(`
select
i.id, i.guid, i.feed_id,
i.title, i.link, i.date,
i.status, i.image, i.podcast_url
select %s
from items i
where %s
order by %s
limit %d
`, predicate, order, limit)
`, selectCols, predicate, order, limit)
rows, err := s.db.Query(query, args...)
if err != nil {
log.Print(err)
@ -178,7 +212,7 @@ func (s *Storage) ListItems(filter ItemFilter, limit int, newestFirst bool) []It
err = rows.Scan(
&x.Id, &x.GUID, &x.FeedId,
&x.Title, &x.Link, &x.Date,
&x.Status, &x.ImageURL, &x.AudioURL,
&x.Status, &x.ImageURL, &x.AudioURL, &x.Content,
)
if err != nil {
log.Print(err)
@ -214,7 +248,11 @@ func (s *Storage) UpdateItemStatus(item_id int64, status ItemStatus) bool {
}
func (s *Storage) MarkItemsRead(filter MarkFilter) bool {
predicate, args := listQueryPredicate(ItemFilter{FolderID: filter.FolderID, FeedID: filter.FeedID}, false)
predicate, args := listQueryPredicate(ItemFilter{
FolderID: filter.FolderID,
FeedID: filter.FeedID,
Before: filter.Before,
}, false)
query := fmt.Sprintf(`
update items as i set status = %d
where %s and i.status != %d

32
src/storage/item_test.go
View File

@ -104,7 +104,7 @@ func TestListItems(t *testing.T) {
// filter by folder_id
have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false))
have := getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder1.Id}, 10, false, false))
want := []string{"item111", "item112", "item113", "item121", "item122"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -112,7 +112,7 @@ func TestListItems(t *testing.T) {
t.Fail()
}
have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false))
have = getItemGuids(db.ListItems(ItemFilter{FolderID: &scope.folder2.Id}, 10, false, false))
want = []string{"item211", "item212"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -122,7 +122,7 @@ func TestListItems(t *testing.T) {
// filter by feed_id
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false))
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed11.Id}, 10, false, false))
want = []string{"item111", "item112", "item113"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -130,7 +130,7 @@ func TestListItems(t *testing.T) {
t.Fail()
}
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false))
have = getItemGuids(db.ListItems(ItemFilter{FeedID: &scope.feed01.Id}, 10, false, false))
want = []string{"item011", "item012", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -141,7 +141,7 @@ func TestListItems(t *testing.T) {
// filter by status
var starred ItemStatus = STARRED
have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false))
have = getItemGuids(db.ListItems(ItemFilter{Status: &starred}, 10, false, false))
want = []string{"item113", "item212", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -150,7 +150,7 @@ func TestListItems(t *testing.T) {
}
var unread ItemStatus = UNREAD
have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false))
have = getItemGuids(db.ListItems(ItemFilter{Status: &unread}, 10, false, false))
want = []string{"item111", "item121", "item011"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -160,7 +160,7 @@ func TestListItems(t *testing.T) {
// limit
have = getItemGuids(db.ListItems(ItemFilter{}, 2, false))
have = getItemGuids(db.ListItems(ItemFilter{}, 2, false, false))
want = []string{"item111", "item112"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -171,7 +171,7 @@ func TestListItems(t *testing.T) {
// filter by search
db.SyncSearch()
search1 := "title111"
have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true))
have = getItemGuids(db.ListItems(ItemFilter{Search: &search1}, 4, true, false))
want = []string{"item111"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -180,7 +180,7 @@ func TestListItems(t *testing.T) {
}
// sort by date
have = getItemGuids(db.ListItems(ItemFilter{}, 4, true))
have = getItemGuids(db.ListItems(ItemFilter{}, 4, true, false))
want = []string{"item013", "item012", "item011", "item212"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -197,7 +197,7 @@ func TestListItemsPaginated(t *testing.T) {
item121 := getItem(db, "item121")
// all, newest first
have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true))
have := getItemGuids(db.ListItems(ItemFilter{After: &item012.Id}, 3, true, false))
want := []string{"item011", "item212", "item211"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -207,7 +207,7 @@ func TestListItemsPaginated(t *testing.T) {
// unread, newest first
unread := UNREAD
have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true))
have = getItemGuids(db.ListItems(ItemFilter{After: &item012.Id, Status: &unread}, 3, true, false))
want = []string{"item011", "item121", "item111"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -217,7 +217,7 @@ func TestListItemsPaginated(t *testing.T) {
// starred, oldest first
starred := STARRED
have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false))
have = getItemGuids(db.ListItems(ItemFilter{After: &item121.Id, Status: &starred}, 3, false, false))
want = []string{"item212", "item013"}
if !reflect.DeepEqual(have, want) {
t.Logf("want: %#v", want)
@ -233,7 +233,7 @@ func TestMarkItemsRead(t *testing.T) {
db1 := testDB()
testItemsSetup(db1)
db1.MarkItemsRead(MarkFilter{})
have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false))
have := getItemGuids(db1.ListItems(ItemFilter{Status: &read}, 10, false, false))
want := []string{
"item111", "item112", "item121", "item122",
"item211", "item011", "item012",
@ -247,7 +247,7 @@ func TestMarkItemsRead(t *testing.T) {
db2 := testDB()
scope2 := testItemsSetup(db2)
db2.MarkItemsRead(MarkFilter{FolderID: &scope2.folder1.Id})
have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false))
have = getItemGuids(db2.ListItems(ItemFilter{Status: &read}, 10, false, false))
want = []string{
"item111", "item112", "item121", "item122",
"item211", "item012",
@ -261,7 +261,7 @@ func TestMarkItemsRead(t *testing.T) {
db3 := testDB()
scope3 := testItemsSetup(db3)
db3.MarkItemsRead(MarkFilter{FeedID: &scope3.feed11.Id})
have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false))
have = getItemGuids(db3.ListItems(ItemFilter{Status: &read}, 10, false, false))
want = []string{
"item111", "item112", "item122",
"item211", "item012",
@ -319,7 +319,7 @@ func TestDeleteOldItems(t *testing.T) {
}
db.DeleteOldItems()
feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false)
feedItems := db.ListItems(ItemFilter{FeedID: &feed.Id}, 1000, false, false)
if len(feedItems) != len(items)-3 {
t.Fatalf(
"invalid number of old items kept\nwant: %d\nhave: %d",