From 4e91ad87bd48be7f88440423f38e77ab3d9957c6 Mon Sep 17 00:00:00 2001 From: makeworld Date: Mon, 10 Aug 2020 18:50:40 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20Adding=20feeds=20&=20pages,=20an?= =?UTF-8?q?d=20JSON=20enc/dec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- amfora.go | 8 +++- config/config.go | 33 +++++++++++-- feeds/feeds.go | 117 +++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 11 +++++ 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 feeds/feeds.go diff --git a/amfora.go b/amfora.go index 34e2e40..0ed153d 100644 --- a/amfora.go +++ b/amfora.go @@ -6,6 +6,7 @@ import ( "github.com/makeworld-the-better-one/amfora/config" "github.com/makeworld-the-better-one/amfora/display" + "github.com/makeworld-the-better-one/amfora/feeds" ) var version = "1.5.0-unreleased" @@ -33,7 +34,12 @@ func main() { err := config.Init() if err != nil { - fmt.Printf("Config error: %v\n", err) + fmt.Fprintf(os.Stderr, "Config error: %v\n", err) + os.Exit(1) + } + err = feeds.Init() + if err != nil { + fmt.Fprintf(os.Stderr, "Config error: %v\n", err) os.Exit(1) } diff --git a/config/config.go b/config/config.go index 38887a4..7d6cf8a 100644 --- a/config/config.go +++ b/config/config.go @@ -1,7 +1,11 @@ +// Package config initializes all files required for Amfora, even those used by +// other packages. It also reads in the config file and initializes a Viper and +// the theme package config import ( "fmt" + "io" "os" "path/filepath" "runtime" @@ -31,9 +35,9 @@ var bkmkPath string var DownloadsDir string // Feeds -var Feeds = viper.New() -var feedsDir string -var feedsPath string +var FeedJson io.ReadCloser +var feedDir string +var FeedPath string func Init() error { @@ -100,6 +104,22 @@ func Init() error { } bkmkPath = filepath.Join(bkmkDir, "bookmarks.toml") + // Feeds dir and path + if runtime.GOOS == "windows" { + // In APPDATA beside other Amfora files + feedDir = amforaAppData + } else { + // XDG data dir on POSIX systems + xdg_data, ok := os.LookupEnv("XDG_DATA_HOME") + if ok && strings.TrimSpace(xdg_data) != "" { + feedDir = filepath.Join(xdg_data, "amfora") + } else { + // Default to ~/.local/share/amfora + feedDir = filepath.Join(home, ".local", "share", "amfora") + } + } + FeedPath = filepath.Join(feedDir, "feeds.json") + // *** Create necessary files and folders *** // Config @@ -135,6 +155,13 @@ func Init() error { if err == nil { f.Close() } + // Feeds + err = os.MkdirAll(feedDir, 0755) + if err != nil { + return err + } + f, _ = os.OpenFile(FeedPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + FeedJson = f // *** Downloads paths, setup, and creation *** diff --git a/feeds/feeds.go b/feeds/feeds.go new file mode 100644 index 0000000..47b69cb --- /dev/null +++ b/feeds/feeds.go @@ -0,0 +1,117 @@ +package feeds + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/makeworld-the-better-one/amfora/config" + "github.com/mmcdole/gofeed" +) + +/* +Example JSON. +{ + "feeds": { + "url1": , + "url2: " + }, + "pages": { + "url1": "hash", + "url2": "hash" + } +} + +"pages" are the pages tracked for changes that aren't feeds. +The hash is SHA-256. + +*/ + +// Decoded JSON +type feedJson struct { + Feeds map[string]*gofeed.Feed `json:"feeds"` + Pages map[string]string `json:"pages"` +} + +var data feedJson + +var ErrSaving = errors.New("couldn't save JSON to disk") + +// Init should be called after config.Init. +func Init() error { + defer config.FeedJson.Close() + + dec := json.NewDecoder(config.FeedJson) + err := dec.Decode(&data) + if err != nil && err != io.EOF { + return fmt.Errorf("feeds json is corrupted: %v", err) + } + return nil +} + +// IsTracked returns true of the feed/page URL is already being tracked. +func IsTracked(url string) bool { + for u := range data.Feeds { + if url == u { + return true + } + } + for u := range data.Pages { + if url == u { + return true + } + } + return false +} + +// GetFeed returns a Feed object and a bool indicating whether the passed +// content was actually recognized as a feed. +func GetFeed(mediatype, filename string, r io.Reader) (*gofeed.Feed, bool) { + // Check mediatype and filename + if mediatype != "application/atom+xml" && mediatype != "application/rss+xml" && + filename != "atom.xml" && filename != "feed.xml" && + !strings.HasSuffix(filename, ".atom") && !strings.HasSuffix(filename, ".rss") { + // No part of the above is true + return nil, false + } + feed, err := gofeed.NewParser().Parse(r) + return feed, err == nil +} + +func writeJson() error { + f, err := os.OpenFile(config.FeedPath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + err = enc.Encode(&data) + return err +} + +// AddFeed stores a feed. +func AddFeed(url string, feed *gofeed.Feed) error { + sort.Sort(feed) + data.Feeds[url] = feed + err := writeJson() + if err != nil { + return ErrSaving + } + return nil +} + +// AddPage stores a page URL to track for changes. +func AddPage(url string) error { + data.Pages[url] = "" // No hash yet + err := writeJson() + if err != nil { + return ErrSaving + } + return nil +} diff --git a/go.mod b/go.mod index 92ee038..9b59051 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/makeworld-the-better-one/progressbar/v3 v3.3.5-0.20200710151429-125743e22b4f github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.3.1 // indirect + github.com/mmcdole/gofeed v1.0.0 github.com/pelletier/go-toml v1.8.0 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cast v1.3.1 // indirect diff --git a/go.sum b/go.sum index be1cd19..e188571 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -27,6 +31,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -152,6 +157,10 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.1 h1:cCBH2gTD2K0OtLlv/Y5H01VQCqmlDxz30kS5Y5bqfLA= github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcdole/gofeed v1.0.0 h1:PHqwr8fsEm8xarj9s53XeEAFYhRM3E9Ib7Ie766/LTE= +github.com/mmcdole/gofeed v1.0.0/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -245,6 +254,7 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -258,6 +268,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=