From 3a667a3809e9d63051d82648877f91a7e6f6cf41 Mon Sep 17 00:00:00 2001 From: Nazar Kanaev Date: Fri, 19 Mar 2021 21:50:07 +0000 Subject: [PATCH] add date parser --- src/feed/date.go | 217 +++++++++++++++++++++++++++++++++++++++++++++ src/feed/json.go | 4 +- src/feed/models.go | 2 +- 3 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 src/feed/date.go diff --git a/src/feed/date.go b/src/feed/date.go new file mode 100644 index 0000000..1f108a5 --- /dev/null +++ b/src/feed/date.go @@ -0,0 +1,217 @@ +package feed + +import ( + "fmt" + "time" +) + +// dateformats taken from somewhere +// which where originally taken from github.com/mjibson/goread +var dateFormats = []string{ + time.RFC822, // RSS + time.RFC822Z, // RSS + time.RFC3339, // Atom + time.UnixDate, + time.RubyDate, + time.RFC850, + time.RFC1123Z, + time.RFC1123, + time.ANSIC, + "Mon, 02 Jan 2006 15:04:05 MST -07:00", + "Mon, January 2, 2006, 3:04 PM MST", + "Mon, January 2 2006 15:04:05 -0700", + "Mon, January 02, 2006, 15:04:05 MST", + "Mon, January 02, 2006 15:04:05 MST", + "Mon, Jan 2, 2006 15:04 MST", + "Mon, Jan 2 2006 15:04 MST", + "Mon, Jan 2 2006 15:04:05 MST", + "Mon, Jan 2, 2006 15:04:05 MST", + "Mon, Jan 2 2006 15:04:05 -700", + "Mon, Jan 2 2006 15:04:05 -0700", + "Mon Jan 2 15:04 2006", + "Mon Jan 2 15:04:05 2006 MST", + "Mon Jan 02, 2006 3:04 pm", + "Mon, Jan 02,2006 15:04:05 MST", + "Mon Jan 02 2006 15:04:05 -0700", + "Mon, 02/01/2006", + "Monday, 2. January 2006 - 15:04", + "Monday 02 January 2006", + "Monday, January 2, 2006 15:04:05 MST", + "Monday, January 2, 2006 03:04 PM", + "Monday, January 2, 2006", + "Monday, January 02, 2006", + "Monday, 2 January 2006 15:04:05 MST", + "Monday, 2 January 2006 15:04:05 -0700", + "Monday, 2 Jan 2006 15:04:05 MST", + "Monday, 2 Jan 2006 15:04:05 -0700", + "Monday, 02 January 2006 15:04:05 MST", + "Monday, 02 January 2006 15:04:05 -0700", + "Monday, 02 January 2006 15:04:05", + "Monday, January 02, 2006 - 3:04pm", + "Monday, January 2, 2006 - 3:04pm", + "Mon, 01/02/2006 - 15:04", + "Mon, 2 January 2006 15:04 MST", + "Mon, 2 January 2006, 15:04 -0700", + "Mon, 2 January 2006, 15:04:05 MST", + "Mon, 2 January 2006 15:04:05 MST", + "Mon, 2 January 2006 15:04:05 -0700", + "Mon, 2 January 2006", + "nilMon, 2 Jan 2006 3:04:05 PM -0700", + "Mon, 2 Jan 2006 15:4:5 MST", + "Mon, 2 Jan 2006 15:4:5 -0700 GMT", + "Mon, 2, Jan 2006 15:4", + "Mon, 2 Jan 2006 15:04 MST", + "Mon, 2 Jan 2006, 15:04 -0700", + "Mon, 2 Jan 2006 15:04 -0700", + "Mon, 2 Jan 2006 15:04:05 UT", + "Mon, 2 Jan 2006 15:04:05MST", + "Mon, 2 Jan 2006 15:04:05 MST", + "Mon 2 Jan 2006 15:04:05 MST", + "mon,2 Jan 2006 15:04:05 MST", + "Mon, 2 Jan 2006 15:04:05 -0700 MST", + "Mon, 2 Jan 2006 15:04:05-0700", + "Mon, 2 Jan 2006 15:04:05 -0700", + "Mon, 2 Jan 2006 15:04:05", + "Mon, 2 Jan 2006 15:04", + "Mon, 02 Jan 2006, 15:04", + "Mon, 2 Jan 2006, 15:04", + "Mon,2 Jan 2006", + "Mon, 2 Jan 2006", + "Mon, 2 Jan 15:04:05 MST", + "Mon, 2 Jan 06 15:04:05 MST", + "Mon, 2 Jan 06 15:04:05 -0700", + "Mon, 2006-01-02 15:04", + "Mon,02 January 2006 14:04:05 MST", + "Mon, 02 January 2006", + "Mon, 02 Jan 2006 3:04:05 PM MST", + "Mon, 02 Jan 2006 15 -0700", + "Mon,02 Jan 2006 15:04 MST", + "Mon, 02 Jan 2006 15:04 MST", + "Mon, 02 Jan 2006 15:04 -0700", + "Mon, 02 Jan 2006 15:04:05 Z", + "Mon, 02 Jan 2006 15:04:05 UT", + "Mon, 02 Jan 2006 15:04:05 MST-07:00", + "Mon, 02 Jan 2006 15:04:05 MST -0700", + "Mon, 02 Jan 2006, 15:04:05 MST", + "Mon, 02 Jan 2006 15:04:05MST", + "Mon, 02 Jan 2006 15:04:05 MST", + "Mon , 02 Jan 2006 15:04:05 MST", + "Mon, 02 Jan 2006 15:04:05 GMT-0700", + "Mon,02 Jan 2006 15:04:05 -0700", + "Mon, 02 Jan 2006 15:04:05 -0700", + "Mon, 02 Jan 2006 15:04:05 -07:00", + "Mon, 02 Jan 2006 15:04:05 --0700", + "Mon 02 Jan 2006 15:04:05 -0700", + "Mon 02 Jan 2006, 15:04:05 MST", + "Mon, 02 Jan 2006 15:04:05 MST", + "Mon, 02 Jan 2006 15:04:05 -07", + "Mon, 02 Jan 2006 15:04:05 00", + "Mon, 02 Jan 2006 15:04:05", + "Mon, 02 Jan 2006", + "Mon, 02 Jan 06 15:04:05 MST", + "Mon, 02 Jan 2006 3:04 PM MST", + "Mon Jan 02 2006 15:04:05 MST", + "Mon, 01 02 2006 15:04:05 -0700", + "Mon, 2th Jan 2006 15:05:05 MST", + "Jan. 2, 2006, 3:04 a.m.", + "fri, 02 jan 2006 15:04:05 -0700", + "January 02 2006 03:04:05 PM", + "January 2, 2006 3:04 PM", + "January 2, 2006, 3:04 p.m.", + "January 2, 2006 15:04:05 MST", + "January 2, 2006 15:04:05", + "January 2, 2006 03:04 PM", + "January 2, 2006", + "January 02, 2006 15:04:05 MST", + "January 02, 2006 15:04", + "January 02, 2006 03:04 PM", + "January 02, 2006", + "Jan 2, 2006 3:04:05 PM MST", + "Jan 2, 2006 3:04:05 PM", + "Jan 2, 2006 15:04:05 MST", + "Jan 2, 2006", + "Jan 02 2006 03:04:05PM", + "Jan 02, 2006", + "6/1/2 15:04", + "6-1-2 15:04", + "2 January 2006 15:04:05 MST", + "2 January 2006 15:04:05 -0700", + "2 January 2006", + "2 Jan 2006 15:04:05 Z", + "2 Jan 2006 15:04:05 MST", + "2 Jan 2006 15:04:05 -0700", + "2 Jan 2006", + "2 Jan 2006 15:04 MST", + "2.1.2006 15:04:05", + "2/1/2006", + "2-1-2006", + "2006 January 02", + "2006-1-2T15:04:05Z", + "2006-1-2 15:04:05", + "2006-1-2", + "2006-01-02T15:04:05-07:00Z", + "2006-1-02T15:04:05Z", + "2006-01-02T15:04Z", + "2006-01-02T15:04-07:00", + "2006-01-02T15:04:05Z", + "2006-01-02T15:04:05-07:00:00", + "2006-01-02T15:04:05:-0700", + "2006-01-02T15:04:05-0700", + "2006-01-02T15:04:05-07:00", + "2006-01-02T15:04:05 -0700", + "2006-01-02T15:04:05:00", + "2006-01-02T15:04:05", + "2006-01-02T15:04", + "2006-01-02 at 15:04:05", + "2006-01-02 15:04:05Z", + "2006-01-02 15:04:05 MST", + "2006-01-02 15:04:05-0700", + "2006-01-02 15:04:05-07:00", + "2006-01-02 15:04:05 -0700", + "2006-01-02 15:04", + "2006-01-02 00:00:00.0 15:04:05.0 -0700", + "2006/01/02", + "2006-01-02", + "15:04 02.01.2006 -0700", + "1/2/2006 3:04 PM MST", + "1/2/2006 3:04:05 PM MST", + "1/2/2006 3:04:05 PM", + "1/2/2006 15:04:05 MST", + "1/2/2006", + "06/1/2 15:04", + "06-1-2 15:04", + "02 Monday, Jan 2006 15:04", + "02 Jan 2006 15:04 MST", + "02 Jan 2006 15:04:05 UT", + "02 Jan 2006 15:04:05 MST", + "02 Jan 2006 15:04:05 -0700", + "02 Jan 2006 15:04:05", + "02 Jan 2006", + "02/01/2006 15:04 MST", + "02-01-2006 15:04:05 MST", + "02.01.2006 15:04:05", + "02/01/2006 15:04:05", + "02.01.2006 15:04", + "02/01/2006 - 15:04", + "02.01.2006 -0700", + "02/01/2006", + "02-01-2006", + "01/02/2006 3:04 PM", + "01/02/2006 15:04:05 MST", + "01/02/2006 - 15:04", + "01/02/2006", + "01-02-2006", + "Jan. 2006", + "Jan. 2, 2006, 03:04 p.m.", + "2006-01-02 15:04:05 -07:00", + "2 January, 2006", +} + +func dateParse(line string) (time.Time, error) { + for _, layout := range dateFormats { + if t, err := time.Parse(layout, line); err == nil { + return t, nil + } + } + return time.Time{}, fmt.Errorf("failed to parse date: %s", line) +} diff --git a/src/feed/json.go b/src/feed/json.go index 7ccd08a..1fbb9a9 100644 --- a/src/feed/json.go +++ b/src/feed/json.go @@ -49,14 +49,14 @@ func (f *jsonFeed) convert(base string) *Feed { FeedURL: f.FeedURL, } for _, item := range f.Items { - //date := item.DatePublished + date, _ := dateParse(first(item.DatePublished, item.DateModified)) content := first(item.HTML, item.Text, item.Summary) imageUrl := "" podcastUrl := "" feed.Items = append(feed.Items, Item{ GUID: item.ID, - Date: nil, + Date: date, URL: item.URL, Title: item.Title, Content: content, diff --git a/src/feed/models.go b/src/feed/models.go index 1a70167..5a40f90 100644 --- a/src/feed/models.go +++ b/src/feed/models.go @@ -11,7 +11,7 @@ type Feed struct { type Item struct { GUID string - Date *time.Time + Date time.Time URL string Title string