From 9367e9b1f2906220eb6c68c639aaf2fed77c6bf1 Mon Sep 17 00:00:00 2001 From: Jebbs Date: Thu, 3 Dec 2020 06:11:24 +0800 Subject: [PATCH] V5: YAML support (#475) * yml support * code optimize * remove external conf loaders * update yaml test code optimize * code optimize * fix a typo * update convert desc --- commands/all/api/inbounds_add.go | 3 +- commands/all/api/inbounds_remove.go | 3 +- commands/all/api/outbounds_add.go | 3 +- commands/all/api/outbounds_remove.go | 3 +- commands/all/api/shared.go | 64 -------- commands/all/commands.go | 1 - commands/all/convert.go | 123 +++++++++++---- commands/all/convert_confs.go | 113 ++++++++++++++ commands/all/format_doc.go | 2 +- commands/all/merge.go | 101 ------------ commands/all/merge_doc.go | 11 +- .../merge/file.go => common/cmdarg/arg.go | 38 +++-- .../cmdarg}/errors.generated.go | 2 +- config.go | 3 +- go.mod | 1 + infra/conf/json/yaml.go | 49 ++++++ infra/conf/json/yaml_test.go | 145 ++++++++++++++++++ infra/conf/merge/merge.go | 21 ++- main/commands/run.go | 2 +- main/confloader/confloader.go | 34 ---- main/confloader/errors.generated.go | 9 -- main/confloader/external/errors.generated.go | 9 -- main/confloader/external/external.go | 87 ----------- main/distro/all/all.go | 11 +- main/json/config_json.go | 38 ----- main/{jsonem/jsonem.go => json/json.go} | 2 +- main/yaml/yaml.go | 69 +++++++++ 27 files changed, 528 insertions(+), 419 deletions(-) create mode 100644 commands/all/convert_confs.go delete mode 100644 commands/all/merge.go rename infra/conf/merge/file.go => common/cmdarg/arg.go (66%) rename {main/jsonem => common/cmdarg}/errors.generated.go (92%) create mode 100644 infra/conf/json/yaml.go create mode 100644 infra/conf/json/yaml_test.go delete mode 100644 main/confloader/confloader.go delete mode 100644 main/confloader/errors.generated.go delete mode 100644 main/confloader/external/errors.generated.go delete mode 100644 main/confloader/external/external.go delete mode 100644 main/json/config_json.go rename main/{jsonem/jsonem.go => json/json.go} (98%) create mode 100644 main/yaml/yaml.go diff --git a/commands/all/api/inbounds_add.go b/commands/all/api/inbounds_add.go index b021c4319..54ad3e7f1 100644 --- a/commands/all/api/inbounds_add.go +++ b/commands/all/api/inbounds_add.go @@ -5,6 +5,7 @@ import ( handlerService "v2ray.com/core/app/proxyman/command" "v2ray.com/core/commands/base" + "v2ray.com/core/common/cmdarg" "v2ray.com/core/infra/conf" "v2ray.com/core/infra/conf/serial" ) @@ -42,7 +43,7 @@ func executeAddInbounds(cmd *base.Command, args []string) { ins := make([]conf.InboundDetourConfig, 0) for _, arg := range unnamedArgs { - r, err := loadArg(arg) + r, err := cmdarg.LoadArg(arg) if err != nil { base.Fatalf("failed to load %s: %s", arg, err) } diff --git a/commands/all/api/inbounds_remove.go b/commands/all/api/inbounds_remove.go index fa9fe8550..a7ad82f9e 100644 --- a/commands/all/api/inbounds_remove.go +++ b/commands/all/api/inbounds_remove.go @@ -5,6 +5,7 @@ import ( handlerService "v2ray.com/core/app/proxyman/command" "v2ray.com/core/commands/base" + "v2ray.com/core/common/cmdarg" "v2ray.com/core/infra/conf/serial" ) @@ -41,7 +42,7 @@ func executeRemoveInbounds(cmd *base.Command, args []string) { tags := make([]string, 0) for _, arg := range unnamedArgs { - if r, err := loadArg(arg); err == nil { + if r, err := cmdarg.LoadArg(arg); err == nil { conf, err := serial.DecodeJSONConfig(r) if err != nil { base.Fatalf("failed to decode %s: %s", arg, err) diff --git a/commands/all/api/outbounds_add.go b/commands/all/api/outbounds_add.go index a31fa620c..a840c8b61 100644 --- a/commands/all/api/outbounds_add.go +++ b/commands/all/api/outbounds_add.go @@ -5,6 +5,7 @@ import ( handlerService "v2ray.com/core/app/proxyman/command" "v2ray.com/core/commands/base" + "v2ray.com/core/common/cmdarg" "v2ray.com/core/infra/conf" "v2ray.com/core/infra/conf/serial" ) @@ -42,7 +43,7 @@ func executeAddOutbounds(cmd *base.Command, args []string) { outs := make([]conf.OutboundDetourConfig, 0) for _, arg := range unnamedArgs { - r, err := loadArg(arg) + r, err := cmdarg.LoadArg(arg) if err != nil { base.Fatalf("failed to load %s: %s", arg, err) } diff --git a/commands/all/api/outbounds_remove.go b/commands/all/api/outbounds_remove.go index 967ec57e8..ddd139f94 100644 --- a/commands/all/api/outbounds_remove.go +++ b/commands/all/api/outbounds_remove.go @@ -5,6 +5,7 @@ import ( handlerService "v2ray.com/core/app/proxyman/command" "v2ray.com/core/commands/base" + "v2ray.com/core/common/cmdarg" "v2ray.com/core/infra/conf/serial" ) @@ -41,7 +42,7 @@ func executeRemoveOutbounds(cmd *base.Command, args []string) { tags := make([]string, 0) for _, arg := range unnamedArgs { - if r, err := loadArg(arg); err == nil { + if r, err := cmdarg.LoadArg(arg); err == nil { conf, err := serial.DecodeJSONConfig(r) if err != nil { base.Fatalf("failed to decode %s: %s", arg, err) diff --git a/commands/all/api/shared.go b/commands/all/api/shared.go index 830034046..2256b7dd1 100644 --- a/commands/all/api/shared.go +++ b/commands/all/api/shared.go @@ -1,21 +1,14 @@ package api import ( - "bytes" "context" "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" "strings" "time" "google.golang.org/grpc" "google.golang.org/protobuf/proto" "v2ray.com/core/commands/base" - "v2ray.com/core/common/buf" ) type serviceHandler func(ctx context.Context, conn *grpc.ClientConn, cmd *base.Command, args []string) string @@ -45,63 +38,6 @@ func dialAPIServer() (conn *grpc.ClientConn, ctx context.Context, close func()) return } -// loadArg loads one arg, maybe an remote url, or local file path -func loadArg(arg string) (out io.Reader, err error) { - var data []byte - switch { - case strings.HasPrefix(arg, "http://"), strings.HasPrefix(arg, "https://"): - data, err = fetchHTTPContent(arg) - - case arg == "stdin:": - data, err = ioutil.ReadAll(os.Stdin) - - default: - data, err = ioutil.ReadFile(arg) - } - - if err != nil { - return - } - out = bytes.NewBuffer(data) - return -} - -// fetchHTTPContent dials https for remote content -func fetchHTTPContent(target string) ([]byte, error) { - parsedTarget, err := url.Parse(target) - if err != nil { - return nil, err - } - - if s := strings.ToLower(parsedTarget.Scheme); s != "http" && s != "https" { - return nil, fmt.Errorf("invalid scheme: %s", parsedTarget.Scheme) - } - - client := &http.Client{ - Timeout: 30 * time.Second, - } - resp, err := client.Do(&http.Request{ - Method: "GET", - URL: parsedTarget, - Close: true, - }) - if err != nil { - return nil, fmt.Errorf("failed to dial to %s", target) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) - } - - content, err := buf.ReadAllToBytes(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read HTTP response") - } - - return content, nil -} - func showResponese(m proto.Message) { msg := "" bs, err := proto.Marshal(m) diff --git a/commands/all/commands.go b/commands/all/commands.go index 8bcd60a7e..58dc3158e 100644 --- a/commands/all/commands.go +++ b/commands/all/commands.go @@ -17,7 +17,6 @@ func init() { tls.CmdTLS, cmdUUID, cmdVerify, - cmdMerge, // documents docFormat, diff --git a/commands/all/convert.go b/commands/all/convert.go index b58b42202..f612523ed 100644 --- a/commands/all/convert.go +++ b/commands/all/convert.go @@ -2,33 +2,54 @@ package all import ( "bytes" + "encoding/json" "os" + "strings" "google.golang.org/protobuf/proto" + "gopkg.in/yaml.v2" "v2ray.com/core/commands/base" - "v2ray.com/core/infra/conf/merge" "v2ray.com/core/infra/conf/serial" ) var cmdConvert = &base.Command{ - UsageLine: "{{.Exec}} convert [-r] [c1.json] [.json] [dir1] ...", - Short: "Convert multiple json config to protobuf", + CustomFlags: true, + UsageLine: "{{.Exec}} convert [c1.json] [.json] [dir1] ...", + Short: "Convert config files", Long: ` -Convert JSON config to protobuf. - -If multiple JSON files or folders specified, it merges them first, then convert. +Convert config files between different formats. Files are merged +before convert if multiple assigned. Arguments: + -i, -input + Specify the input format. + Available values: "json", "yaml" + Default: "json" + + -o, -output + Specify the output format + Available values: "json", "yaml", "protobuf" / "pb" + Default: "json" + -r Load confdir recursively. Examples: - {{.Exec}} {{.LongName}} config.json - {{.Exec}} {{.LongName}} c1.json c2.json - {{.Exec}} {{.LongName}} c1.json https://url.to/c2.json - {{.Exec}} {{.LongName}} "path/to/json_dir" + {{.Exec}} {{.LongName}} -output=protobuf config.json (1) + {{.Exec}} {{.LongName}} -output=yaml config.json (2) + {{.Exec}} {{.LongName}} -input=yaml config.yaml (3) + {{.Exec}} {{.LongName}} "path/to/dir" (4) + {{.Exec}} {{.LongName}} -i yaml -o protobuf c1.yaml .yaml (5) + +(1) Convert json to protobuf +(2) Convert json to yaml +(3) Convert yaml to json +(4) Merge json files in dir +(5) Merge yaml files and convert to protobuf + +Use "{{.Exec}} help config-merge" for more information about merge. `, } @@ -36,36 +57,76 @@ func init() { cmdConvert.Run = executeConvert // break init loop } -var convertReadDirRecursively = cmdConvert.Flag.Bool("r", false, "") +var ( + inputFormat string + outputFormat string + confDirRecursively bool +) +var formatExtensions = map[string][]string{ + "json": {".json", ".jsonc"}, + "yaml": {".yaml", ".yml"}, +} +func setConfArgs(cmd *base.Command) { + cmd.Flag.StringVar(&inputFormat, "input", "json", "") + cmd.Flag.StringVar(&inputFormat, "i", "json", "") + cmd.Flag.StringVar(&outputFormat, "output", "json", "") + cmd.Flag.StringVar(&outputFormat, "o", "json", "") + cmd.Flag.BoolVar(&confDirRecursively, "r", true, "") +} func executeConvert(cmd *base.Command, args []string) { + setConfArgs(cmd) + cmd.Flag.Parse(args) unnamed := cmd.Flag.Args() - files := resolveFolderToFiles(unnamed, *convertReadDirRecursively) + inputFormat = strings.ToLower(inputFormat) + outputFormat = strings.ToLower(outputFormat) + + files := resolveFolderToFiles(unnamed, formatExtensions[inputFormat], confDirRecursively) if len(files) == 0 { base.Fatalf("empty config list") } + m := mergeConvertToMap(files, inputFormat) - data, err := merge.FilesToJSON(files) - if err != nil { - base.Fatalf("failed to load json: %s", err) - } - r := bytes.NewReader(data) - cf, err := serial.DecodeJSONConfig(r) - if err != nil { - base.Fatalf("failed to decode json: %s", err) + var ( + out []byte + err error + ) + switch outputFormat { + case "json": + out, err = json.Marshal(m) + if err != nil { + base.Fatalf("failed to marshal json: %s", err) + } + case "yaml": + out, err = yaml.Marshal(m) + if err != nil { + base.Fatalf("failed to marshal json: %s", err) + } + case "pb", "protobuf": + data, err := json.Marshal(m) + if err != nil { + base.Fatalf("failed to marshal json: %s", err) + } + r := bytes.NewReader(data) + cf, err := serial.DecodeJSONConfig(r) + if err != nil { + base.Fatalf("failed to decode json: %s", err) + } + pbConfig, err := cf.Build() + if err != nil { + base.Fatalf(err.Error()) + } + out, err = proto.Marshal(pbConfig) + if err != nil { + base.Fatalf("failed to marshal proto config: %s", err) + } + default: + base.Errorf("invalid output format: %s", outputFormat) + base.Errorf("Run '%s help %s' for details.", base.CommandEnv.Exec, cmd.LongName()) + base.Exit() } - pbConfig, err := cf.Build() - if err != nil { - base.Fatalf(err.Error()) - } - - bytesConfig, err := proto.Marshal(pbConfig) - if err != nil { - base.Fatalf("failed to marshal proto config: %s", err) - } - - if _, err := os.Stdout.Write(bytesConfig); err != nil { + if _, err := os.Stdout.Write(out); err != nil { base.Fatalf("failed to write proto config: %s", err) } } diff --git a/commands/all/convert_confs.go b/commands/all/convert_confs.go new file mode 100644 index 000000000..72bebd9f9 --- /dev/null +++ b/commands/all/convert_confs.go @@ -0,0 +1,113 @@ +package all + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + "v2ray.com/core/commands/base" + "v2ray.com/core/common/cmdarg" + "v2ray.com/core/infra/conf/json" + "v2ray.com/core/infra/conf/merge" +) + +func mergeConvertToMap(files []string, format string) map[string]interface{} { + var ( + m map[string]interface{} + err error + ) + switch inputFormat { + case "json": + m, err = merge.FilesToMap(files) + if err != nil { + base.Fatalf("failed to load json: %s", err) + } + case "yaml": + bs, err := yamlsToJSONs(files) + if err != nil { + base.Fatalf("failed to convert yaml to json: %s", err) + } + m, err = merge.BytesToMap(bs) + if err != nil { + base.Fatalf("failed to merge converted json: %s", err) + } + default: + base.Errorf("invalid input format: %s", format) + base.Errorf("Run '%s help %s' for details.", base.CommandEnv.Exec, cmdConvert.LongName()) + base.Exit() + } + return m +} + +// resolveFolderToFiles expands folder path (if any and it exists) to file paths. +// Any other paths, like file, even URL, it returns them as is. +func resolveFolderToFiles(paths []string, extensions []string, recursively bool) []string { + dirReader := readConfDir + if recursively { + dirReader = readConfDirRecursively + } + files := make([]string, 0) + for _, p := range paths { + i, err := os.Stat(p) + if err == nil && i.IsDir() { + files = append(files, dirReader(p, extensions)...) + continue + } + files = append(files, p) + } + return files +} + +func readConfDir(dirPath string, extensions []string) []string { + confs, err := ioutil.ReadDir(dirPath) + if err != nil { + base.Fatalf("failed to read dir %s: %s", dirPath, err) + } + files := make([]string, 0) + for _, f := range confs { + ext := filepath.Ext(f.Name()) + for _, e := range extensions { + if strings.EqualFold(ext, e) { + files = append(files, filepath.Join(dirPath, f.Name())) + break + } + } + } + return files +} + +// getFolderFiles get files in the folder and it's children +func readConfDirRecursively(dirPath string, extensions []string) []string { + files := make([]string, 0) + err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + ext := filepath.Ext(path) + for _, e := range extensions { + if strings.EqualFold(ext, e) { + files = append(files, path) + break + } + } + return nil + }) + if err != nil { + base.Fatalf("failed to read dir %s: %s", dirPath, err) + } + return files +} + +func yamlsToJSONs(files []string) ([][]byte, error) { + jsons := make([][]byte, 0) + for _, file := range files { + bs, err := cmdarg.LoadArgToBytes(file) + if err != nil { + return nil, err + } + j, err := json.FromYAML(bs) + if err != nil { + return nil, err + } + jsons = append(jsons, j) + } + return jsons, nil +} diff --git a/commands/all/format_doc.go b/commands/all/format_doc.go index 341f6efc0..ca1024533 100644 --- a/commands/all/format_doc.go +++ b/commands/all/format_doc.go @@ -14,7 +14,7 @@ var docFormat = &base.Command{ The default loader, multiple config files support. * yaml (.yml) - The yaml loader (coming soon?), multiple config files support. + The yaml loader, multiple config files support. * protobuf / pb (.pb) Single conifg file support. If multiple files assigned, diff --git a/commands/all/merge.go b/commands/all/merge.go deleted file mode 100644 index 3424f29d9..000000000 --- a/commands/all/merge.go +++ /dev/null @@ -1,101 +0,0 @@ -package all - -import ( - "io/ioutil" - "os" - "path/filepath" - - "v2ray.com/core/commands/base" - "v2ray.com/core/infra/conf/merge" -) - -var cmdMerge = &base.Command{ - UsageLine: "{{.Exec}} merge [-r] [c1.json] [url] [dir1] ...", - Short: "Merge json files into one", - Long: ` -Merge JSON files into one. - -Arguments: - - -r - Load confdir recursively. - -Examples: - - {{.Exec}} {{.LongName}} c1.json c2.json - {{.Exec}} {{.LongName}} c1.json https://url.to/c2.json - {{.Exec}} {{.LongName}} "path/to/json_dir" -`, -} - -func init() { - cmdMerge.Run = executeMerge -} - -var mergeReadDirRecursively = cmdMerge.Flag.Bool("r", false, "") - -func executeMerge(cmd *base.Command, args []string) { - unnamed := cmd.Flag.Args() - files := resolveFolderToFiles(unnamed, *mergeReadDirRecursively) - if len(files) == 0 { - base.Fatalf("empty config list") - } - - data, err := merge.FilesToJSON(files) - if err != nil { - base.Fatalf(err.Error()) - } - if _, err := os.Stdout.Write(data); err != nil { - base.Fatalf(err.Error()) - } -} - -// resolveFolderToFiles expands folder path (if any and it exists) to file paths. -// Any other paths, like file, even URL, it returns them as is. -func resolveFolderToFiles(paths []string, recursively bool) []string { - dirReader := readConfDir - if recursively { - dirReader = readConfDirRecursively - } - files := make([]string, 0) - for _, p := range paths { - i, err := os.Stat(p) - if err == nil && i.IsDir() { - files = append(files, dirReader(p)...) - continue - } - files = append(files, p) - } - return files -} - -func readConfDir(dirPath string) []string { - confs, err := ioutil.ReadDir(dirPath) - if err != nil { - base.Fatalf("failed to read dir %s: %s", dirPath, err) - } - files := make([]string, 0) - for _, f := range confs { - ext := filepath.Ext(f.Name()) - if ext == ".json" || ext == ".jsonc" { - files = append(files, filepath.Join(dirPath, f.Name())) - } - } - return files -} - -// getFolderFiles get files in the folder and it's children -func readConfDirRecursively(dirPath string) []string { - files := make([]string, 0) - err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { - ext := filepath.Ext(path) - if ext == ".json" || ext == ".jsonc" { - files = append(files, path) - } - return nil - }) - if err != nil { - base.Fatalf("failed to read dir %s: %s", dirPath, err) - } - return files -} diff --git a/commands/all/merge_doc.go b/commands/all/merge_doc.go index 692549d0d..a601c74e8 100644 --- a/commands/all/merge_doc.go +++ b/commands/all/merge_doc.go @@ -5,15 +5,18 @@ import ( ) var docMerge = &base.Command{ - UsageLine: "{{.Exec}} json-merge", - Short: "json merge logic", + UsageLine: "{{.Exec}} config-merge", + Short: "config merge logic", Long: ` -Merging of JSON configs is applied in following commands: +Merging of config files is applied in following commands: {{.Exec}} run -c c1.json -c c2.json ... - {{.Exec}} merge c1.json https://url.to/c2.json ... + {{.Exec}} test -c c1.yaml -c c2.yaml ... {{.Exec}} convert c1.json dir1 ... +Support of yaml is implemented by converting yaml to json, +both merge and load. So we take json as example here. + Suppose we have 2 JSON files, The 1st one: diff --git a/infra/conf/merge/file.go b/common/cmdarg/arg.go similarity index 66% rename from infra/conf/merge/file.go rename to common/cmdarg/arg.go index 26c711494..8d6dfaa6a 100644 --- a/infra/conf/merge/file.go +++ b/common/cmdarg/arg.go @@ -1,4 +1,4 @@ -package merge +package cmdarg import ( "bytes" @@ -11,29 +11,36 @@ import ( "time" "v2ray.com/core/common/buf" - "v2ray.com/core/infra/conf/serial" ) -// loadArg loads one arg, maybe an remote url, or local file path -func loadArg(arg string) (out io.Reader, err error) { - var data []byte +// LoadArg loads one arg, maybe an remote url, or local file path +func LoadArg(arg string) (out io.Reader, err error) { + bs, err := LoadArgToBytes(arg) + if err != nil { + return nil, err + } + out = bytes.NewBuffer(bs) + return +} + +// LoadArgToBytes loads one arg to []byte, maybe an remote url, or local file path +func LoadArgToBytes(arg string) (out []byte, err error) { switch { case strings.HasPrefix(arg, "http://"), strings.HasPrefix(arg, "https://"): - data, err = fetchHTTPContent(arg) + out, err = FetchHTTPContent(arg) case (arg == "stdin:"): - data, err = ioutil.ReadAll(os.Stdin) + out, err = ioutil.ReadAll(os.Stdin) default: - data, err = ioutil.ReadFile(arg) + out, err = ioutil.ReadFile(arg) } if err != nil { return } - out = bytes.NewBuffer(data) return } -// fetchHTTPContent dials https for remote content -func fetchHTTPContent(target string) ([]byte, error) { +// FetchHTTPContent dials https for remote content +func FetchHTTPContent(target string) ([]byte, error) { parsedTarget, err := url.Parse(target) if err != nil { return nil, newError("invalid URL: ", target).Base(err) @@ -67,12 +74,3 @@ func fetchHTTPContent(target string) ([]byte, error) { return content, nil } - -func decode(r io.Reader) (map[string]interface{}, error) { - c := make(map[string]interface{}) - err := serial.DecodeJSON(r, &c) - if err != nil { - return nil, err - } - return c, nil -} diff --git a/main/jsonem/errors.generated.go b/common/cmdarg/errors.generated.go similarity index 92% rename from main/jsonem/errors.generated.go rename to common/cmdarg/errors.generated.go index 48961bcea..71477f44f 100644 --- a/main/jsonem/errors.generated.go +++ b/common/cmdarg/errors.generated.go @@ -1,4 +1,4 @@ -package jsonem +package cmdarg import "v2ray.com/core/common/errors" diff --git a/config.go b/config.go index 8c6016fad..4c1089a55 100644 --- a/config.go +++ b/config.go @@ -11,7 +11,6 @@ import ( "v2ray.com/core/common" "v2ray.com/core/common/buf" "v2ray.com/core/common/cmdarg" - "v2ray.com/core/main/confloader" ) // ConfigFormat is a configurable format of V2Ray config file. @@ -107,7 +106,7 @@ func init() { Loader: func(input interface{}) (*Config, error) { switch v := input.(type) { case cmdarg.Arg: - r, err := confloader.LoadConfig(v[0]) + r, err := cmdarg.LoadArg(v[0]) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index f3597d562..1c67c8fb4 100644 --- a/go.mod +++ b/go.mod @@ -21,5 +21,6 @@ require ( golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 google.golang.org/grpc v1.33.2 google.golang.org/protobuf v1.25.0 + gopkg.in/yaml.v2 v2.3.0 h12.io/socks v1.0.1 ) diff --git a/infra/conf/json/yaml.go b/infra/conf/json/yaml.go new file mode 100644 index 000000000..55180573d --- /dev/null +++ b/infra/conf/json/yaml.go @@ -0,0 +1,49 @@ +package json + +import ( + "encoding/json" + "fmt" + + "gopkg.in/yaml.v2" +) + +// FromYAML convert yaml to json +func FromYAML(v []byte) ([]byte, error) { + m1 := make(map[interface{}]interface{}) + err := yaml.Unmarshal(v, &m1) + if err != nil { + return nil, err + } + m2 := convert(m1) + j, err := json.Marshal(m2) + if err != nil { + return nil, err + } + return j, nil +} + +func convert(m map[interface{}]interface{}) map[string]interface{} { + res := map[string]interface{}{} + for k, v := range m { + var value interface{} + switch v2 := v.(type) { + case map[interface{}]interface{}: + value = convert(v2) + case []interface{}: + for i, el := range v2 { + if m, ok := el.(map[interface{}]interface{}); ok { + v2[i] = convert(m) + } + } + value = v2 + default: + value = v + } + key := "null" + if k != nil { + key = fmt.Sprint(k) + } + res[key] = value + } + return res +} diff --git a/infra/conf/json/yaml_test.go b/infra/conf/json/yaml_test.go new file mode 100644 index 000000000..84ada0a86 --- /dev/null +++ b/infra/conf/json/yaml_test.go @@ -0,0 +1,145 @@ +package json + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestYMLToJSON_V2Style(t *testing.T) { + input := ` +log: + loglevel: debug +inbounds: +- port: 10800 + listen: 127.0.0.1 + protocol: socks + settings: + udp: true +outbounds: +- protocol: vmess + settings: + vnext: + - address: example.com + port: 443 + users: + - id: '98a15fa6-2eb1-edd5-50ea-cfc428aaab78' + streamSettings: + network: tcp + security: tls +` + expected := ` +{ + "log": { + "loglevel": "debug" + }, + "inbounds": [{ + "port": 10800, + "listen": "127.0.0.1", + "protocol": "socks", + "settings": { + "udp": true + } + }], + "outbounds": [{ + "protocol": "vmess", + "settings": { + "vnext": [{ + "port": 443, + "address": "example.com", + "users": [{ + "id": "98a15fa6-2eb1-edd5-50ea-cfc428aaab78" + }] + }] + }, + "streamSettings": { + "network": "tcp", + "security": "tls" + } + }] +} +` + bs, err := FromYAML([]byte(input)) + if err != nil { + t.Error(err) + } + m := make(map[string]interface{}) + json.Unmarshal(bs, &m) + assertResult(t, m, expected) +} +func TestYMLToJSON_ValueTypes(t *testing.T) { + input := ` +boolean: + - TRUE + - FALSE + - true + - false +float: + - 3.14 + - 6.8523015e+5 +int: + - 123 + - 0b1010_0111_0100_1010_1110 +null: + nodeName: 'node' + parent: ~ # ~ for null +string: + - 哈哈 + - 'Hello world' + - newline + newline2 # multi-line string +date: + - 2018-02-17 # yyyy-MM-dd +datetime: + - 2018-02-17T15:02:31+08:00 # ISO 8601 time +mixed: + - true + - false + - 1 + - 0 + - null + - hello +# arbitrary keys +1: 0 +true: false +TRUE: TRUE +"str": "hello" +` + expected := ` +{ + "boolean": [true, false, true, false], + "float": [3.14, 685230.15], + "int": [123, 685230], + "null": { + "nodeName": "node", + "parent": null + }, + "string": ["哈哈", "Hello world", "newline newline2"], + "date": ["2018-02-17"], + "datetime": ["2018-02-17T15:02:31+08:00"], + "mixed": [true,false,1,0,null,"hello"], + "1": 0, + "true": true, + "str": "hello" +} +` + bs, err := FromYAML([]byte(input)) + if err != nil { + t.Error(err) + } + m := make(map[string]interface{}) + json.Unmarshal(bs, &m) + assertResult(t, m, expected) +} + +func assertResult(t *testing.T, value map[string]interface{}, expected string) { + e := make(map[string]interface{}) + err := json.Unmarshal([]byte(expected), &e) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(value, e) { + bs, _ := json.Marshal(value) + t.Fatalf("expected:\n%s\n\nactual:\n%s", expected, string(bs)) + } +} diff --git a/infra/conf/merge/merge.go b/infra/conf/merge/merge.go index 3d2badd7b..cdc39097d 100644 --- a/infra/conf/merge/merge.go +++ b/infra/conf/merge/merge.go @@ -18,6 +18,10 @@ package merge import ( "bytes" "encoding/json" + "io" + + "v2ray.com/core/common/cmdarg" + "v2ray.com/core/infra/conf/serial" ) // FilesToJSON merges multiple jsons files into one json, accepts remote url, or local file path @@ -65,9 +69,9 @@ func BytesToMap(args [][]byte) (m map[string]interface{}, err error) { } func loadFiles(args []string) (map[string]interface{}, error) { - conf := make(map[string]interface{}) + c := make(map[string]interface{}) for _, arg := range args { - r, err := loadArg(arg) + r, err := cmdarg.LoadArg(arg) if err != nil { return nil, err } @@ -75,11 +79,11 @@ func loadFiles(args []string) (map[string]interface{}, error) { if err != nil { return nil, err } - if err = mergeMaps(conf, m); err != nil { + if err = mergeMaps(c, m); err != nil { return nil, err } } - return conf, nil + return c, nil } func loadBytes(args [][]byte) (map[string]interface{}, error) { @@ -96,3 +100,12 @@ func loadBytes(args [][]byte) (map[string]interface{}, error) { } return conf, nil } + +func decode(r io.Reader) (map[string]interface{}, error) { + c := make(map[string]interface{}) + err := serial.DecodeJSON(r, &c) + if err != nil { + return nil, err + } + return c, nil +} diff --git a/main/commands/run.go b/main/commands/run.go index 20e14f968..6ce56f693 100644 --- a/main/commands/run.go +++ b/main/commands/run.go @@ -56,7 +56,7 @@ var ( ) func setConfigFlags(cmd *base.Command) { - configFormat = cmd.Flag.String("format", "json", "") + configFormat = cmd.Flag.String("format", "", "") configDirRecursively = cmd.Flag.Bool("r", false, "") cmd.Flag.Var(&configFiles, "config", "") diff --git a/main/confloader/confloader.go b/main/confloader/confloader.go deleted file mode 100644 index 24da1a0cb..000000000 --- a/main/confloader/confloader.go +++ /dev/null @@ -1,34 +0,0 @@ -package confloader - -import ( - "io" - "os" -) - -type configFileLoader func(string) (io.Reader, error) -type extconfigLoader func([]string, io.Reader) (io.Reader, error) - -var ( - EffectiveConfigFileLoader configFileLoader - EffectiveExtConfigLoader extconfigLoader -) - -// LoadConfig reads from a path/url/stdin -// actual work is in external module -func LoadConfig(file string) (io.Reader, error) { - if EffectiveConfigFileLoader == nil { - newError("external config module not loaded, reading from stdin").AtInfo().WriteToLog() - return os.Stdin, nil - } - return EffectiveConfigFileLoader(file) -} - -// LoadExtConfig calls v2ctl to handle multiple config -// the actual work also in external module -func LoadExtConfig(files []string, reader io.Reader) (io.Reader, error) { - if EffectiveExtConfigLoader == nil { - return nil, newError("external config module not loaded").AtError() - } - - return EffectiveExtConfigLoader(files, reader) -} diff --git a/main/confloader/errors.generated.go b/main/confloader/errors.generated.go deleted file mode 100644 index deda6e515..000000000 --- a/main/confloader/errors.generated.go +++ /dev/null @@ -1,9 +0,0 @@ -package confloader - -import "v2ray.com/core/common/errors" - -type errPathObjHolder struct{} - -func newError(values ...interface{}) *errors.Error { - return errors.New(values...).WithPathObj(errPathObjHolder{}) -} diff --git a/main/confloader/external/errors.generated.go b/main/confloader/external/errors.generated.go deleted file mode 100644 index 919f10d00..000000000 --- a/main/confloader/external/errors.generated.go +++ /dev/null @@ -1,9 +0,0 @@ -package external - -import "v2ray.com/core/common/errors" - -type errPathObjHolder struct{} - -func newError(values ...interface{}) *errors.Error { - return errors.New(values...).WithPathObj(errPathObjHolder{}) -} diff --git a/main/confloader/external/external.go b/main/confloader/external/external.go deleted file mode 100644 index e7b64693f..000000000 --- a/main/confloader/external/external.go +++ /dev/null @@ -1,87 +0,0 @@ -package external - -//go:generate go run v2ray.com/core/common/errors/errorgen - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "strings" - "time" - - "v2ray.com/core/common/buf" - "v2ray.com/core/common/platform/ctlcmd" - "v2ray.com/core/main/confloader" -) - -func ConfigLoader(arg string) (out io.Reader, err error) { - var data []byte - switch { - case strings.HasPrefix(arg, "http://"), strings.HasPrefix(arg, "https://"): - data, err = FetchHTTPContent(arg) - - case arg == "stdin:": - data, err = ioutil.ReadAll(os.Stdin) - - default: - data, err = ioutil.ReadFile(arg) - } - - if err != nil { - return - } - out = bytes.NewBuffer(data) - return -} - -func FetchHTTPContent(target string) ([]byte, error) { - parsedTarget, err := url.Parse(target) - if err != nil { - return nil, newError("invalid URL: ", target).Base(err) - } - - if s := strings.ToLower(parsedTarget.Scheme); s != "http" && s != "https" { - return nil, newError("invalid scheme: ", parsedTarget.Scheme) - } - - client := &http.Client{ - Timeout: 30 * time.Second, - } - resp, err := client.Do(&http.Request{ - Method: "GET", - URL: parsedTarget, - Close: true, - }) - if err != nil { - return nil, newError("failed to dial to ", target).Base(err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, newError("unexpected HTTP status code: ", resp.StatusCode) - } - - content, err := buf.ReadAllToBytes(resp.Body) - if err != nil { - return nil, newError("failed to read HTTP response").Base(err) - } - - return content, nil -} - -func ExtConfigLoader(files []string, reader io.Reader) (io.Reader, error) { - buf, err := ctlcmd.Run(append([]string{"convert"}, files...), reader) - if err != nil { - return nil, err - } - - return strings.NewReader(buf.String()), nil -} - -func init() { - confloader.EffectiveConfigFileLoader = ConfigLoader - confloader.EffectiveExtConfigLoader = ExtConfigLoader -} diff --git a/main/distro/all/all.go b/main/distro/all/all.go index 29992676c..6cd06dddb 100644 --- a/main/distro/all/all.go +++ b/main/distro/all/all.go @@ -55,14 +55,11 @@ import ( _ "v2ray.com/core/transport/internet/headers/wechat" _ "v2ray.com/core/transport/internet/headers/wireguard" - // JSON config support. Choose only one from the two below. - // The following line loads JSON from v2ctl - // _ "v2ray.com/core/main/json" - // The following line loads JSON internally - _ "v2ray.com/core/main/jsonem" + // JSON config support. + _ "v2ray.com/core/main/json" - // Load config from file or http(s) - _ "v2ray.com/core/main/confloader/external" + // YAML config support. + _ "v2ray.com/core/main/yaml" // commands _ "v2ray.com/core/commands/all" diff --git a/main/json/config_json.go b/main/json/config_json.go deleted file mode 100644 index d5c9e1510..000000000 --- a/main/json/config_json.go +++ /dev/null @@ -1,38 +0,0 @@ -package json - -//go:generate go run v2ray.com/core/common/errors/errorgen - -import ( - "io" - "os" - - "v2ray.com/core" - "v2ray.com/core/common" - "v2ray.com/core/common/cmdarg" - "v2ray.com/core/main/confloader" -) - -func init() { - common.Must(core.RegisterConfigLoader(&core.ConfigFormat{ - Name: []string{"JSON"}, - Extension: []string{".json", ".jsonc"}, - Loader: func(input interface{}) (*core.Config, error) { - switch v := input.(type) { - case cmdarg.Arg: - r, err := confloader.LoadExtConfig(v, os.Stdin) - if err != nil { - return nil, newError("failed to execute v2ctl to convert config file.").Base(err).AtWarning() - } - return core.LoadConfig("protobuf", "", r) - case io.Reader: - r, err := confloader.LoadExtConfig([]string{"stdin:"}, os.Stdin) - if err != nil { - return nil, newError("failed to execute v2ctl to convert config file.").Base(err).AtWarning() - } - return core.LoadConfig("protobuf", "", r) - default: - return nil, newError("unknown type") - } - }, - })) -} diff --git a/main/jsonem/jsonem.go b/main/json/json.go similarity index 98% rename from main/jsonem/jsonem.go rename to main/json/json.go index 82d3973c7..024f28871 100644 --- a/main/jsonem/jsonem.go +++ b/main/json/json.go @@ -1,4 +1,4 @@ -package jsonem +package json import ( "bytes" diff --git a/main/yaml/yaml.go b/main/yaml/yaml.go new file mode 100644 index 000000000..f5cf46e8a --- /dev/null +++ b/main/yaml/yaml.go @@ -0,0 +1,69 @@ +package yaml + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + + "v2ray.com/core" + "v2ray.com/core/common" + "v2ray.com/core/common/cmdarg" + "v2ray.com/core/infra/conf/json" + "v2ray.com/core/infra/conf/merge" + "v2ray.com/core/infra/conf/serial" +) + +func init() { + common.Must(core.RegisterConfigLoader(&core.ConfigFormat{ + Name: []string{"YAML"}, + Extension: []string{".yml", ".yaml"}, + Loader: func(input interface{}) (*core.Config, error) { + switch v := input.(type) { + case cmdarg.Arg: + bs, err := yamlsToJSONs(v) + if err != nil { + return nil, err + } + data, err := merge.BytesToJSON(bs) + if err != nil { + return nil, err + } + r := bytes.NewReader(data) + cf, err := serial.DecodeJSONConfig(r) + if err != nil { + return nil, err + } + return cf.Build() + case io.Reader: + bs, err := ioutil.ReadAll(v) + if err != nil { + return nil, err + } + bs, err = json.FromYAML(bs) + if err != nil { + return nil, err + } + return serial.LoadJSONConfig(bytes.NewBuffer(bs)) + default: + return nil, errors.New("unknow type") + } + }, + })) +} + +func yamlsToJSONs(files []string) ([][]byte, error) { + jsons := make([][]byte, 0) + for _, file := range files { + bs, err := cmdarg.LoadArgToBytes(file) + if err != nil { + return nil, err + } + j, err := json.FromYAML(bs) + if err != nil { + return nil, err + } + jsons = append(jsons, j) + } + return jsons, nil +}