🔀 Merge charsets branch, fixes #3

README.md

@@ -19,7 +19,7 @@ Amfora aims to be the best looking [Gemini](https://gemini.circumlunar.space/) c
 
 It also aims to be completely cross platform, with full Windows support. If you're on Windows, I would not recommend using the default terminal software. Maybe use [Cmder](https://cmder.net/) instead?
 
-It fully passes Sean Conman's client torture test, with exception of the alternate encodings section, as only UTF-8 and ASCII are supported. It mostly passes the Egsam test.
+It fully passes Sean Conman's client torture test. It mostly passes the Egsam test.
 
 ## Installation
 
@@ -41,6 +41,7 @@ It is designed with large or fullscreen terminals in mind. For optimal usage, ma
 - [x] Styled page content (headings, links)
 - [x] Basic forward/backward history, for each tab
 - [x] Input (Status Code 10 & 11)
+- [x] Multiple charset support (over 55)
 - [ ] Built-in search using GUS
 - [ ] Bookmarks
 - [ ] Search in pages with <kbd>Ctrl-F</kbd>

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.6.0
 	gitlab.com/tslocum/cview v1.4.8-0.20200614211415-f477be8ba472
+	golang.org/x/text v0.3.3
 	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200603094226-e3079894b1e8 // indirect
 )

go.sum

@@ -290,6 +290,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

renderer/renderer.go

@@ -17,10 +17,19 @@ import (
 	"github.com/makeworld-the-better-one/go-gemini"
 	"github.com/spf13/viper"
 	"gitlab.com/tslocum/cview"
+	"golang.org/x/text/encoding/ianaindex"
 )
 
-// Charset values that are compatible with UTF-8, lowercased.
-var utfCharsets = []string{"", "utf-8", "us-ascii"}
+// Charsets that are compatible with UTF-8 and don't need to be decoded.
+func isUTF8(charset string) bool {
+	utfCharsets := []string{"", "utf-8", "us-ascii"}
+	for i := range utfCharsets {
+		if strings.ToLower(charset) == utfCharsets[i] {
+			return true
+		}
+	}
+	return false
+}
 
 // CanDisplay returns true if the response is supported by Amfora
 // for displaying on the screen.
@@ -38,12 +47,12 @@ func CanDisplay(res *gemini.Response) bool {
 		// Amfora doesn't support other filetypes
 		return false
 	}
-	for _, charset := range utfCharsets {
-		if strings.ToLower(params["charset"]) == charset {
-			return true // Supported
-		}
+	if isUTF8(params["charset"]) {
+		return true
 	}
-	return false
+	enc, err := ianaindex.MIME.Encoding(params["charset"]) // Lowercasing is done inside
+	// Encoding sometimes returns nil, see #3 on this repo and golang/go#19421
+	return err == nil && enc != nil
 }
 
 // convertRegularGemini converts non-preformatted blocks of text/gemini
@@ -211,23 +220,39 @@ func MakePage(url string, res *gemini.Response) (*structs.Page, error) {
 		return nil, errors.New("not valid content for a Page")
 	}
 
-	content, err := ioutil.ReadAll(res.Body) // TODO: Don't use all memory on large pages
+	rawText, err := ioutil.ReadAll(res.Body) // TODO: Don't use all memory on large pages
 	if err != nil {
 		return nil, err
 	}
 	res.Body.Close()
 
-	mediatype, _, _ := mime.ParseMediaType(res.Meta)
+	mediatype, params, _ := mime.ParseMediaType(res.Meta)
+
+	// Convert content first
+	var utfText string
+	if isUTF8(params["charset"]) {
+		utfText = string(rawText)
+	} else {
+		encoding, err := ianaindex.MIME.Encoding(params["charset"])
+		if encoding == nil || err != nil {
+			// Some encoding doesn't exist and wasn't caught in CanDisplay()
+			return nil, errors.New("unsupported encoding")
+		}
+		utfText, err = encoding.NewDecoder().String(string(rawText))
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	if mediatype == "text/plain" {
 		return &structs.Page{
 			Url:     url,
-			Content: string(content),
+			Content: utfText,
 			Links:   []string{}, // Plaintext has no links
 		}, nil
 	}
 	if mediatype == "text/gemini" {
-		rendered, links := RenderGemini(string(content))
+		rendered, links := RenderGemini(utfText)
 		return &structs.Page{
 			Url:     url,
 			Content: rendered,