From 1e378fced2c5a34c94fd52335a0bd3e21e227932 Mon Sep 17 00:00:00 2001 From: makeworld Date: Fri, 7 Aug 2020 12:27:50 -0400 Subject: [PATCH 01/33] =?UTF-8?q?=F0=9F=8E=A8=20Reorder=20config.Init=20co?= =?UTF-8?q?de?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 +- README.md | 4 +- config/config.go | 108 +++++++++++++++++++++++++++-------------------- config/theme.go | 4 +- 4 files changed, 69 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4be83a0..98364e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Emoji favicons can now be seen if `emoji_favicons` is enabled in the config (#62) +- **Feed & page subscription** (#61) +- **Emoji favicons** can now be seen if `emoji_favicons` is enabled in the config (#62) - The `shift_numbers` key in the config was added, so that non US keyboard users can navigate tabs (#64) - F1 and F2 keys for navigating to the previous and next tabs (#64) diff --git a/README.md b/README.md index 243b639..ffbae0c 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,8 @@ Features in *italics* are in the master branch, but not in the latest release. - [x] Theming - [x] *Emoji favicons* - See `gemini://mozz.us/files/rfc_gemini_favicon.gmi` for details -- [ ] Subscribe to RSS and Atom feeds and display them - - Subscribing to page changes, similar to how Spacewalk works, will also be supported +- [x] *Subscribe to RSS and Atom feeds and display them* + - Subscribing to page changes, similar to how Spacewalk works, is also supported - [ ] Stream support - [ ] Full client certificate UX within the client - Create transient and permanent certs within the client, per domain diff --git a/config/config.go b/config/config.go index 528b7c4..38887a4 100644 --- a/config/config.go +++ b/config/config.go @@ -30,7 +30,15 @@ var bkmkPath string // For other pkgs to use var DownloadsDir string +// Feeds +var Feeds = viper.New() +var feedsDir string +var feedsPath string + func Init() error { + + // *** Set paths *** + home, err := homedir.Dir() if err != nil { return err @@ -92,7 +100,7 @@ func Init() error { } bkmkPath = filepath.Join(bkmkDir, "bookmarks.toml") - // Create necessary files and folders + // *** Create necessary files and folders *** // Config err = os.MkdirAll(configDir, 0755) @@ -114,56 +122,21 @@ func Init() error { if err != nil { return err } - os.OpenFile(tofuDBPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + f, err = os.OpenFile(tofuDBPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err == nil { + f.Close() + } // Bookmarks err = os.MkdirAll(bkmkDir, 0755) if err != nil { return err } - os.OpenFile(bkmkPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) - - // Setup vipers - - TofuStore.SetConfigFile(tofuDBPath) - TofuStore.SetConfigType("toml") - err = TofuStore.ReadInConfig() - if err != nil { - return err + f, err = os.OpenFile(bkmkPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err == nil { + f.Close() } - BkmkStore.SetConfigFile(bkmkPath) - BkmkStore.SetConfigType("toml") - err = BkmkStore.ReadInConfig() - if err != nil { - return err - } - BkmkStore.Set("DO NOT TOUCH", true) - err = BkmkStore.WriteConfig() - if err != nil { - return err - } - - viper.SetDefault("a-general.home", "gemini.circumlunar.space") - viper.SetDefault("a-general.http", "default") - viper.SetDefault("a-general.search", "gus.guru/search") - viper.SetDefault("a-general.color", true) - viper.SetDefault("a-general.bullets", true) - viper.SetDefault("a-general.left_margin", 0.15) - viper.SetDefault("a-general.max_width", 100) - viper.SetDefault("a-general.downloads", "") - viper.SetDefault("a-general.page_max_size", 2097152) - viper.SetDefault("a-general.page_max_time", 10) - viper.SetDefault("a-general.emoji_favicons", false) - viper.SetDefault("keybindings.shift_numbers", "!@#$%^&*()") - viper.SetDefault("cache.max_size", 0) - viper.SetDefault("cache.max_pages", 20) - - viper.SetConfigFile(configPath) - viper.SetConfigType("toml") - err = viper.ReadInConfig() - if err != nil { - return err - } + // *** Downloads paths, setup, and creation *** // Setup downloads dir if viper.GetString("a-general.downloads") == "" { @@ -196,11 +169,56 @@ func Init() error { DownloadsDir = dDir } + // *** Setup vipers *** + + TofuStore.SetConfigFile(tofuDBPath) + TofuStore.SetConfigType("toml") + err = TofuStore.ReadInConfig() + if err != nil { + return err + } + + BkmkStore.SetConfigFile(bkmkPath) + BkmkStore.SetConfigType("toml") + err = BkmkStore.ReadInConfig() + if err != nil { + return err + } + BkmkStore.Set("DO NOT TOUCH", true) + err = BkmkStore.WriteConfig() + if err != nil { + return err + } + + // Setup main config + + viper.SetDefault("a-general.home", "gemini.circumlunar.space") + viper.SetDefault("a-general.http", "default") + viper.SetDefault("a-general.search", "gus.guru/search") + viper.SetDefault("a-general.color", true) + viper.SetDefault("a-general.bullets", true) + viper.SetDefault("a-general.left_margin", 0.15) + viper.SetDefault("a-general.max_width", 100) + viper.SetDefault("a-general.downloads", "") + viper.SetDefault("a-general.page_max_size", 2097152) + viper.SetDefault("a-general.page_max_time", 10) + viper.SetDefault("a-general.emoji_favicons", false) + viper.SetDefault("keybindings.shift_numbers", "!@#$%^&*()") + viper.SetDefault("cache.max_size", 0) + viper.SetDefault("cache.max_pages", 20) + + viper.SetConfigFile(configPath) + viper.SetConfigType("toml") + err = viper.ReadInConfig() + if err != nil { + return err + } + // Setup cache from config cache.SetMaxSize(viper.GetInt("cache.max_size")) cache.SetMaxPages(viper.GetInt("cache.max_pages")) - // Theme + // Setup theme configTheme := viper.Sub("theme") if configTheme != nil { for k, v := range configTheme.AllSettings() { diff --git a/config/theme.go b/config/theme.go index 6c021cc..7de75f9 100644 --- a/config/theme.go +++ b/config/theme.go @@ -8,7 +8,7 @@ import ( ) // Functions to allow themeing configuration. -// UI element colors are mapped to a string key, such as "error" or "tab_background" +// UI element colors are mapped to a string key, such as "error" or "tab_bg" // These are the same keys used in the config file. var themeMu = sync.RWMutex{} @@ -64,8 +64,8 @@ var theme = map[string]tcell.Color{ func SetColor(key string, color tcell.Color) { themeMu.Lock() - defer themeMu.Unlock() theme[key] = color + themeMu.Unlock() } // GetColor will return tcell.ColorBlack if there is no color for the provided key. From 4e91ad87bd48be7f88440423f38e77ab3d9957c6 Mon Sep 17 00:00:00 2001 From: makeworld Date: Mon, 10 Aug 2020 18:50:40 -0400 Subject: [PATCH 02/33] =?UTF-8?q?=F0=9F=9A=A7=20Adding=20feeds=20&=20pages?= =?UTF-8?q?,=20and=20JSON=20enc/dec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- amfora.go | 8 +++- config/config.go | 33 +++++++++++-- feeds/feeds.go | 117 +++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 11 +++++ 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 feeds/feeds.go diff --git a/amfora.go b/amfora.go index 34e2e40..0ed153d 100644 --- a/amfora.go +++ b/amfora.go @@ -6,6 +6,7 @@ import ( "github.com/makeworld-the-better-one/amfora/config" "github.com/makeworld-the-better-one/amfora/display" + "github.com/makeworld-the-better-one/amfora/feeds" ) var version = "1.5.0-unreleased" @@ -33,7 +34,12 @@ func main() { err := config.Init() if err != nil { - fmt.Printf("Config error: %v\n", err) + fmt.Fprintf(os.Stderr, "Config error: %v\n", err) + os.Exit(1) + } + err = feeds.Init() + if err != nil { + fmt.Fprintf(os.Stderr, "Config error: %v\n", err) os.Exit(1) } diff --git a/config/config.go b/config/config.go index 38887a4..7d6cf8a 100644 --- a/config/config.go +++ b/config/config.go @@ -1,7 +1,11 @@ +// Package config initializes all files required for Amfora, even those used by +// other packages. It also reads in the config file and initializes a Viper and +// the theme package config import ( "fmt" + "io" "os" "path/filepath" "runtime" @@ -31,9 +35,9 @@ var bkmkPath string var DownloadsDir string // Feeds -var Feeds = viper.New() -var feedsDir string -var feedsPath string +var FeedJson io.ReadCloser +var feedDir string +var FeedPath string func Init() error { @@ -100,6 +104,22 @@ func Init() error { } bkmkPath = filepath.Join(bkmkDir, "bookmarks.toml") + // Feeds dir and path + if runtime.GOOS == "windows" { + // In APPDATA beside other Amfora files + feedDir = amforaAppData + } else { + // XDG data dir on POSIX systems + xdg_data, ok := os.LookupEnv("XDG_DATA_HOME") + if ok && strings.TrimSpace(xdg_data) != "" { + feedDir = filepath.Join(xdg_data, "amfora") + } else { + // Default to ~/.local/share/amfora + feedDir = filepath.Join(home, ".local", "share", "amfora") + } + } + FeedPath = filepath.Join(feedDir, "feeds.json") + // *** Create necessary files and folders *** // Config @@ -135,6 +155,13 @@ func Init() error { if err == nil { f.Close() } + // Feeds + err = os.MkdirAll(feedDir, 0755) + if err != nil { + return err + } + f, _ = os.OpenFile(FeedPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + FeedJson = f // *** Downloads paths, setup, and creation *** diff --git a/feeds/feeds.go b/feeds/feeds.go new file mode 100644 index 0000000..47b69cb --- /dev/null +++ b/feeds/feeds.go @@ -0,0 +1,117 @@ +package feeds + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + + "github.com/makeworld-the-better-one/amfora/config" + "github.com/mmcdole/gofeed" +) + +/* +Example JSON. +{ + "feeds": { + "url1": , + "url2: " + }, + "pages": { + "url1": "hash", + "url2": "hash" + } +} + +"pages" are the pages tracked for changes that aren't feeds. +The hash is SHA-256. + +*/ + +// Decoded JSON +type feedJson struct { + Feeds map[string]*gofeed.Feed `json:"feeds"` + Pages map[string]string `json:"pages"` +} + +var data feedJson + +var ErrSaving = errors.New("couldn't save JSON to disk") + +// Init should be called after config.Init. +func Init() error { + defer config.FeedJson.Close() + + dec := json.NewDecoder(config.FeedJson) + err := dec.Decode(&data) + if err != nil && err != io.EOF { + return fmt.Errorf("feeds json is corrupted: %v", err) + } + return nil +} + +// IsTracked returns true of the feed/page URL is already being tracked. +func IsTracked(url string) bool { + for u := range data.Feeds { + if url == u { + return true + } + } + for u := range data.Pages { + if url == u { + return true + } + } + return false +} + +// GetFeed returns a Feed object and a bool indicating whether the passed +// content was actually recognized as a feed. +func GetFeed(mediatype, filename string, r io.Reader) (*gofeed.Feed, bool) { + // Check mediatype and filename + if mediatype != "application/atom+xml" && mediatype != "application/rss+xml" && + filename != "atom.xml" && filename != "feed.xml" && + !strings.HasSuffix(filename, ".atom") && !strings.HasSuffix(filename, ".rss") { + // No part of the above is true + return nil, false + } + feed, err := gofeed.NewParser().Parse(r) + return feed, err == nil +} + +func writeJson() error { + f, err := os.OpenFile(config.FeedPath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return err + } + defer f.Close() + enc := json.NewEncoder(f) + enc.SetEscapeHTML(false) + enc.SetIndent("", " ") + err = enc.Encode(&data) + return err +} + +// AddFeed stores a feed. +func AddFeed(url string, feed *gofeed.Feed) error { + sort.Sort(feed) + data.Feeds[url] = feed + err := writeJson() + if err != nil { + return ErrSaving + } + return nil +} + +// AddPage stores a page URL to track for changes. +func AddPage(url string) error { + data.Pages[url] = "" // No hash yet + err := writeJson() + if err != nil { + return ErrSaving + } + return nil +} diff --git a/go.mod b/go.mod index 92ee038..9b59051 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/makeworld-the-better-one/progressbar/v3 v3.3.5-0.20200710151429-125743e22b4f github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.3.1 // indirect + github.com/mmcdole/gofeed v1.0.0 github.com/pelletier/go-toml v1.8.0 // indirect github.com/spf13/afero v1.2.2 // indirect github.com/spf13/cast v1.3.1 // indirect diff --git a/go.sum b/go.sum index be1cd19..e188571 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= +github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o= +github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -27,6 +31,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -152,6 +157,10 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.1 h1:cCBH2gTD2K0OtLlv/Y5H01VQCqmlDxz30kS5Y5bqfLA= github.com/mitchellh/mapstructure v1.3.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mmcdole/gofeed v1.0.0 h1:PHqwr8fsEm8xarj9s53XeEAFYhRM3E9Ib7Ie766/LTE= +github.com/mmcdole/gofeed v1.0.0/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI= +github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -245,6 +254,7 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -258,6 +268,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= From d254917b3639637e662ab4cd5289844087396c03 Mon Sep 17 00:00:00 2001 From: makeworld Date: Sun, 16 Aug 2020 17:42:45 -0400 Subject: [PATCH 03/33] =?UTF-8?q?=F0=9F=9A=A7=20Sorting,=20update=20funcs,?= =?UTF-8?q?=20mutexes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- amfora.go | 2 +- feeds/feeds.go | 156 ++++++++++++++++++++++++++++++++++++++--------- feeds/structs.go | 83 +++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 29 deletions(-) create mode 100644 feeds/structs.go diff --git a/amfora.go b/amfora.go index 0ed153d..263f586 100644 --- a/amfora.go +++ b/amfora.go @@ -39,7 +39,7 @@ func main() { } err = feeds.Init() if err != nil { - fmt.Fprintf(os.Stderr, "Config error: %v\n", err) + fmt.Fprintf(os.Stderr, "feeds.json error: %v\n", err) os.Exit(1) } diff --git a/feeds/feeds.go b/feeds/feeds.go index 47b69cb..d496a7d 100644 --- a/feeds/feeds.go +++ b/feeds/feeds.go @@ -1,45 +1,32 @@ package feeds import ( + "crypto/sha256" "encoding/json" "errors" "fmt" "io" + "mime" "os" + "path" "sort" "strings" + "sync" + "time" + "github.com/makeworld-the-better-one/amfora/client" "github.com/makeworld-the-better-one/amfora/config" + "github.com/makeworld-the-better-one/go-gemini" "github.com/mmcdole/gofeed" ) -/* -Example JSON. -{ - "feeds": { - "url1": , - "url2: " - }, - "pages": { - "url1": "hash", - "url2": "hash" - } -} +var ( + ErrSaving = errors.New("couldn't save JSON to disk") + ErrNotSuccess = errors.New("status 20 not returned") + ErrNotFeed = errors.New("not a valid feed") +) -"pages" are the pages tracked for changes that aren't feeds. -The hash is SHA-256. - -*/ - -// Decoded JSON -type feedJson struct { - Feeds map[string]*gofeed.Feed `json:"feeds"` - Pages map[string]string `json:"pages"` -} - -var data feedJson - -var ErrSaving = errors.New("couldn't save JSON to disk") +var writeMu = sync.Mutex{} // Init should be called after config.Init. func Init() error { @@ -51,20 +38,28 @@ func Init() error { return fmt.Errorf("feeds json is corrupted: %v", err) } return nil + + // TODO: Start pulling all feeds in another thread } -// IsTracked returns true of the feed/page URL is already being tracked. +// IsTracked returns true if the feed/page URL is already being tracked. func IsTracked(url string) bool { + data.feedMu.RLock() for u := range data.Feeds { if url == u { + data.feedMu.RUnlock() return true } } + data.feedMu.RUnlock() + data.pageMu.RLock() for u := range data.Pages { if url == u { + data.pageMu.RUnlock() return true } } + data.pageMu.RUnlock() return false } @@ -79,10 +74,16 @@ func GetFeed(mediatype, filename string, r io.Reader) (*gofeed.Feed, bool) { return nil, false } feed, err := gofeed.NewParser().Parse(r) + if feed == nil { + return nil, false + } return feed, err == nil } func writeJson() error { + writeMu.Lock() + defer writeMu.Unlock() + f, err := os.OpenFile(config.FeedPath, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { return err @@ -91,27 +92,126 @@ func writeJson() error { enc := json.NewEncoder(f) enc.SetEscapeHTML(false) enc.SetIndent("", " ") + + data.feedMu.Lock() + data.pageMu.Lock() err = enc.Encode(&data) + data.feedMu.Unlock() + data.pageMu.Unlock() + return err } // AddFeed stores a feed. +// It can be used to update a feed for a URL, although the package +// will handle that on its own. func AddFeed(url string, feed *gofeed.Feed) error { + if feed == nil { + panic("feed is nil") + } + sort.Sort(feed) + // Remove any content to save memory and disk space + for _, item := range feed.Items { + item.Content = "" + } + + data.feedMu.Lock() data.Feeds[url] = feed + data.feedMu.Unlock() + err := writeJson() if err != nil { + // Don't use in-memory if it couldn't be saved + data.feedMu.Lock() + delete(data.Feeds, url) + data.feedMu.Unlock() return ErrSaving } return nil } // AddPage stores a page URL to track for changes. +// Do not use it to update a page, as it only resets the hash. func AddPage(url string) error { - data.Pages[url] = "" // No hash yet + data.pageMu.Lock() + data.Pages[url] = &pageJson{} // No hash yet + data.pageMu.Unlock() + err := writeJson() if err != nil { + // Don't use in-memory if it couldn't be saved + data.pageMu.Lock() + delete(data.Pages, url) + data.pageMu.Unlock() return ErrSaving } return nil } + +func updateFeed(url string) error { + res, err := client.Fetch(url) + if err != nil { + if res != nil { + res.Body.Close() + } + return err + } + defer res.Body.Close() + + if res.Status != gemini.StatusSuccess { + return ErrNotSuccess + } + mediatype, _, err := mime.ParseMediaType(res.Meta) + if err != nil { + return err + } + filename := path.Base(url) + feed, ok := GetFeed(mediatype, filename, res.Body) + if !ok { + return ErrNotFeed + } + return AddFeed(url, feed) +} + +func updatePage(url string) error { + res, err := client.Fetch(url) + if err != nil { + if res != nil { + res.Body.Close() + } + return err + } + defer res.Body.Close() + + if res.Status != gemini.StatusSuccess { + return ErrNotSuccess + } + h := sha256.New() + if _, err := io.Copy(h, res.Body); err != nil { + return err + } + data.pageMu.Lock() + data.Pages[url] = &pageJson{ + Hash: fmt.Sprintf("%x", h.Sum(nil)), + Updated: time.Now().UTC(), + } + data.pageMu.Unlock() + + err = writeJson() + if err != nil { + // Don't use in-memory if it couldn't be saved + data.pageMu.Lock() + delete(data.Pages, url) + data.pageMu.Unlock() + return err + } + + return nil +} + +// updateAll updates all feeds and pages. +// It should run in goroutine at a regular interval. +func updateAll() { + +} diff --git a/feeds/structs.go b/feeds/structs.go new file mode 100644 index 0000000..01e2ac4 --- /dev/null +++ b/feeds/structs.go @@ -0,0 +1,83 @@ +package feeds + +import ( + "sync" + "time" + + "github.com/mmcdole/gofeed" +) + +/* +Example JSON. +{ + "feeds": { + "url1": , + "url2": , + }, + "pages": { + "url1": { + "hash": , + "updated":