V5: Add TOML Support (#564)

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/gorilla/websocket v1.4.2
 	github.com/lucas-clemente/quic-go v0.19.3
 	github.com/miekg/dns v1.1.35
+	github.com/pelletier/go-toml v1.8.1
 	github.com/pires/go-proxyproto v0.3.3
 	github.com/seiflotfy/cuckoofilter v0.0.0-20201009151232-afb285a456ab
 	github.com/stretchr/testify v1.6.1

go.sum

@@ -123,6 +123,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
+github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM=
+github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/pires/go-proxyproto v0.3.3 h1:jOXGrsAfSQVFiD1hWg1aiHpLYsd6SJw/8cLN594sB7Q=

infra/conf/json/toml.go

@@ -0,0 +1,20 @@
+package json
+
+import (
+	"encoding/json"
+
+	"github.com/pelletier/go-toml"
+)
+
+// FromTOML convert toml to json
+func FromTOML(v []byte) ([]byte, error) {
+	m := make(map[string]interface{})
+	if err := toml.Unmarshal(v, &m); err != nil {
+		return nil, err
+	}
+	j, err := json.Marshal(m)
+	if err != nil {
+		return nil, err
+	}
+	return j, nil
+}

infra/conf/json/toml_test.go

@@ -0,0 +1,113 @@
+package json
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestTOMLToJSON_V2Style(t *testing.T) {
+	input := `
+[log]
+loglevel = 'debug'
+
+[[inbounds]]
+port = 10800
+listen = '127.0.0.1'
+protocol = 'socks'
+
+[inbounds.settings]
+udp = true
+
+[[outbounds]]
+protocol = 'vmess'
+[[outbounds.settings.vnext]]
+port = 443
+address = 'example.com'
+
+[[outbounds.settings.vnext.users]]
+id = '98a15fa6-2eb1-edd5-50ea-cfc428aaab78'
+
+[outbounds.streamSettings]
+network = 'tcp'
+security = 'tls'
+`
+	expected := `
+{
+    "log": {
+        "loglevel": "debug"
+    },
+    "inbounds": [{
+        "port": 10800,
+        "listen": "127.0.0.1",
+        "protocol": "socks",
+        "settings": {
+            "udp": true
+        }
+    }],
+    "outbounds": [{
+        "protocol": "vmess",
+        "settings": {
+            "vnext": [{
+                "port": 443,
+                "address": "example.com",
+                "users": [{
+                    "id": "98a15fa6-2eb1-edd5-50ea-cfc428aaab78"
+                }]
+            }]
+        },
+        "streamSettings": {
+            "network": "tcp",
+            "security": "tls"
+        }
+    }]
+}
+`
+	bs, err := FromTOML([]byte(input))
+	if err != nil {
+		t.Error(err)
+	}
+	m := make(map[string]interface{})
+	json.Unmarshal(bs, &m)
+	assertResult(t, m, expected)
+}
+func TestTOMLToJSON_ValueTypes(t *testing.T) {
+	input := `
+boolean = [ true, false, true, false ]
+float = [ 3.14, 685_230.15 ]
+int = [ 123, 685_230 ]
+string = [ "哈哈", "Hello world", "newline newline2" ]
+date = [ "2018-02-17" ]
+datetime = [ "2018-02-17T15:02:31+08:00" ]
+mixed = [ true, false, 1, 0, "hello" ]
+1 = 0
+true = true
+str = "hello"
+
+[null]
+nodeName = "node"
+`
+	expected := `
+{
+    "boolean": [true, false, true, false],
+    "float": [3.14, 685230.15],
+    "int": [123, 685230],
+    "null": {
+        "nodeName": "node"
+    },
+    "string": ["哈哈", "Hello world",  "newline newline2"],
+    "date": ["2018-02-17"],
+    "datetime": ["2018-02-17T15:02:31+08:00"],
+    "mixed": [true,false,1,0,"hello"],
+    "1": 0,
+    "true": true,
+    "str": "hello"
+}
+`
+	bs, err := FromTOML([]byte(input))
+	if err != nil {
+		t.Error(err)
+	}
+	m := make(map[string]interface{})
+	json.Unmarshal(bs, &m)
+	assertResult(t, m, expected)
+}

main/commands/all/convert.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/pelletier/go-toml"
 	"google.golang.org/protobuf/proto"
 	"gopkg.in/yaml.v2"
 	"v2ray.com/core/infra/conf/serial"
@@ -24,12 +25,12 @@ Arguments:
 
 	-i, -input
 		Specify the input format.
-		Available values: "json", "yaml"
+		Available values: "json", "toml", "yaml"
 		Default: "json"
 
 	-o, -output
 		Specify the output format
-		Available values: "json", "yaml", "protobuf" / "pb"
+		Available values: "json", "toml", "yaml", "protobuf" / "pb"
 		Default: "json"
 
 	-r
@@ -38,16 +39,14 @@ Arguments:
 Examples:
 
 	{{.Exec}} {{.LongName}} -output=protobuf config.json           (1)
-	{{.Exec}} {{.LongName}} -output=yaml config.json               (2)
+	{{.Exec}} {{.LongName}} -input=toml config.toml                (2)
-	{{.Exec}} {{.LongName}} -input=yaml config.yaml                (3)
+	{{.Exec}} {{.LongName}} "path/to/dir"                          (3)
-	{{.Exec}} {{.LongName}} "path/to/dir"                          (4)
+	{{.Exec}} {{.LongName}} -i yaml -o protobuf c1.yaml <url>.yaml (4)
-	{{.Exec}} {{.LongName}} -i yaml -o protobuf c1.yaml <url>.yaml (5)
 
 (1) Convert json to protobuf
-(2) Convert json to yaml
+(2) Convert toml to json
-(3) Convert yaml to json
+(3) Merge json files in dir
-(4) Merge json files in dir
+(4) Merge yaml files and convert to protobuf
-(5) Merge yaml files and convert to protobuf
 
 Use "{{.Exec}} help config-merge" for more information about merge.
 `,
@@ -64,6 +63,7 @@ var (
 )
 var formatExtensions = map[string][]string{
 	"json": {".json", ".jsonc"},
+	"toml": {".toml"},
 	"yaml": {".yaml", ".yml"},
 }
 
@@ -97,6 +97,11 @@ func executeConvert(cmd *base.Command, args []string) {
 		if err != nil {
 			base.Fatalf("failed to marshal json: %s", err)
 		}
+	case "toml":
+		out, err = toml.Marshal(m)
+		if err != nil {
+			base.Fatalf("failed to marshal json: %s", err)
+		}
 	case "yaml":
 		out, err = yaml.Marshal(m)
 		if err != nil {

main/commands/all/convert_confs.go

@@ -23,6 +23,15 @@ func mergeConvertToMap(files []string, format string) map[string]interface{} {
 		if err != nil {
 			base.Fatalf("failed to load json: %s", err)
 		}
+	case "toml":
+		bs, err := tomlsToJSONs(files)
+		if err != nil {
+			base.Fatalf("failed to convert toml to json: %s", err)
+		}
+		m, err = merge.BytesToMap(bs)
+		if err != nil {
+			base.Fatalf("failed to merge converted json: %s", err)
+		}
 	case "yaml":
 		bs, err := yamlsToJSONs(files)
 		if err != nil {
@@ -111,3 +120,19 @@ func yamlsToJSONs(files []string) ([][]byte, error) {
 	}
 	return jsons, nil
 }
+
+func tomlsToJSONs(files []string) ([][]byte, error) {
+	jsons := make([][]byte, 0)
+	for _, file := range files {
+		bs, err := cmdarg.LoadArgToBytes(file)
+		if err != nil {
+			return nil, err
+		}
+		j, err := json.FromTOML(bs)
+		if err != nil {
+			return nil, err
+		}
+		jsons = append(jsons, j)
+	}
+	return jsons, nil
+}

main/commands/all/format_doc.go

@@ -11,7 +11,10 @@ var docFormat = &base.Command{
 {{.Exec}} supports different config formats:
 
 	* json (.json, .jsonc)
-	  The default loader, multiple config files support.
+	  The default loader, multiple config files support.	
+
+	* toml (.toml)
+	  The toml loader, multiple config files support.
 
 	* yaml (.yml)
 	  The yaml loader, multiple config files support.

main/commands/all/merge_doc.go

@@ -14,7 +14,7 @@ Merging of config files is applied in following commands:
 	{{.Exec}} test -c c1.yaml -c c2.yaml ...
 	{{.Exec}} convert c1.json dir1 ...
 
-Support of yaml is implemented by converting yaml to json, 
+Support of toml and yaml is implemented by converting them to json, 
 both merge and load. So we take json as example here.
 
 Suppose we have 2 JSON files,

main/distro/all/all.go

@@ -58,6 +58,9 @@ import (
 	// JSON config support.
 	_ "v2ray.com/core/main/json"
 
+	// TOML config support.
+	_ "v2ray.com/core/main/toml"
+
 	// YAML config support.
 	_ "v2ray.com/core/main/yaml"
 

main/toml/toml.go

@@ -0,0 +1,69 @@
+package toml
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+
+	"v2ray.com/core"
+	"v2ray.com/core/common"
+	"v2ray.com/core/common/cmdarg"
+	"v2ray.com/core/infra/conf/json"
+	"v2ray.com/core/infra/conf/merge"
+	"v2ray.com/core/infra/conf/serial"
+)
+
+func init() {
+	common.Must(core.RegisterConfigLoader(&core.ConfigFormat{
+		Name:      []string{"TOML"},
+		Extension: []string{".toml"},
+		Loader: func(input interface{}) (*core.Config, error) {
+			switch v := input.(type) {
+			case cmdarg.Arg:
+				bs, err := tomlsToJSONs(v)
+				if err != nil {
+					return nil, err
+				}
+				data, err := merge.BytesToJSON(bs)
+				if err != nil {
+					return nil, err
+				}
+				r := bytes.NewReader(data)
+				cf, err := serial.DecodeJSONConfig(r)
+				if err != nil {
+					return nil, err
+				}
+				return cf.Build()
+			case io.Reader:
+				bs, err := ioutil.ReadAll(v)
+				if err != nil {
+					return nil, err
+				}
+				bs, err = json.FromTOML(bs)
+				if err != nil {
+					return nil, err
+				}
+				return serial.LoadJSONConfig(bytes.NewBuffer(bs))
+			default:
+				return nil, errors.New("unknow type")
+			}
+		},
+	}))
+}
+
+func tomlsToJSONs(files []string) ([][]byte, error) {
+	jsons := make([][]byte, 0)
+	for _, file := range files {
+		bs, err := cmdarg.LoadArgToBytes(file)
+		if err != nil {
+			return nil, err
+		}
+		j, err := json.FromTOML(bs)
+		if err != nil {
+			return nil, err
+		}
+		jsons = append(jsons, j)
+	}
+	return jsons, nil
+}