diff --git a/commands/all/api/api.go b/commands/all/api/api.go index d76b89e94..b5ab1c7d4 100644 --- a/commands/all/api/api.go +++ b/commands/all/api/api.go @@ -1,7 +1,7 @@ package api import ( - "v2ray.com/core/commands/base" + "github.com/v2fly/v2ray-core/v4/commands/base" ) // CmdAPI calls an API in an V2Ray process diff --git a/commands/all/api/inbounds_add.go b/commands/all/api/inbounds_add.go index b021c4319..05f5ae11f 100644 --- a/commands/all/api/inbounds_add.go +++ b/commands/all/api/inbounds_add.go @@ -3,10 +3,11 @@ package api import ( "fmt" - handlerService "v2ray.com/core/app/proxyman/command" - "v2ray.com/core/commands/base" - "v2ray.com/core/infra/conf" - "v2ray.com/core/infra/conf/serial" + handlerService "github.com/v2fly/v2ray-core/v4/app/proxyman/command" + "github.com/v2fly/v2ray-core/v4/commands/base" + "github.com/v2fly/v2ray-core/v4/common/cmdarg" + "github.com/v2fly/v2ray-core/v4/infra/conf" + "github.com/v2fly/v2ray-core/v4/infra/conf/serial" ) var cmdAddInbounds = &base.Command{ @@ -42,7 +43,7 @@ func executeAddInbounds(cmd *base.Command, args []string) { ins := make([]conf.InboundDetourConfig, 0) for _, arg := range unnamedArgs { - r, err := loadArg(arg) + r, err := cmdarg.LoadArg(arg) if err != nil { base.Fatalf("failed to load %s: %s", arg, err) } diff --git a/commands/all/api/inbounds_remove.go b/commands/all/api/inbounds_remove.go index fa9fe8550..376e6d57a 100644 --- a/commands/all/api/inbounds_remove.go +++ b/commands/all/api/inbounds_remove.go @@ -3,9 +3,10 @@ package api import ( "fmt" - handlerService "v2ray.com/core/app/proxyman/command" - "v2ray.com/core/commands/base" - "v2ray.com/core/infra/conf/serial" + handlerService "github.com/v2fly/v2ray-core/v4/app/proxyman/command" + "github.com/v2fly/v2ray-core/v4/commands/base" + "github.com/v2fly/v2ray-core/v4/common/cmdarg" + "github.com/v2fly/v2ray-core/v4/infra/conf/serial" ) var cmdRemoveInbounds = &base.Command{ @@ -41,7 +42,7 @@ func executeRemoveInbounds(cmd *base.Command, args []string) { tags := make([]string, 0) for _, arg := range unnamedArgs { - if r, err := loadArg(arg); err == nil { + if r, err := cmdarg.LoadArg(arg); err == nil { conf, err := serial.DecodeJSONConfig(r) if err != nil { base.Fatalf("failed to decode %s: %s", arg, err) diff --git a/commands/all/api/logger_restart.go b/commands/all/api/logger_restart.go index e6de4f74a..cebdd8254 100644 --- a/commands/all/api/logger_restart.go +++ b/commands/all/api/logger_restart.go @@ -1,8 +1,8 @@ package api import ( - logService "v2ray.com/core/app/log/command" - "v2ray.com/core/commands/base" + logService "github.com/v2fly/v2ray-core/v4/app/log/command" + "github.com/v2fly/v2ray-core/v4/commands/base" ) var cmdRestartLogger = &base.Command{ diff --git a/commands/all/api/outbounds_add.go b/commands/all/api/outbounds_add.go index a31fa620c..6d658f914 100644 --- a/commands/all/api/outbounds_add.go +++ b/commands/all/api/outbounds_add.go @@ -3,10 +3,11 @@ package api import ( "fmt" - handlerService "v2ray.com/core/app/proxyman/command" - "v2ray.com/core/commands/base" - "v2ray.com/core/infra/conf" - "v2ray.com/core/infra/conf/serial" + handlerService "github.com/v2fly/v2ray-core/v4/app/proxyman/command" + "github.com/v2fly/v2ray-core/v4/commands/base" + "github.com/v2fly/v2ray-core/v4/common/cmdarg" + "github.com/v2fly/v2ray-core/v4/infra/conf" + "github.com/v2fly/v2ray-core/v4/infra/conf/serial" ) var cmdAddOutbounds = &base.Command{ @@ -42,7 +43,7 @@ func executeAddOutbounds(cmd *base.Command, args []string) { outs := make([]conf.OutboundDetourConfig, 0) for _, arg := range unnamedArgs { - r, err := loadArg(arg) + r, err := cmdarg.LoadArg(arg) if err != nil { base.Fatalf("failed to load %s: %s", arg, err) } diff --git a/commands/all/api/outbounds_remove.go b/commands/all/api/outbounds_remove.go index 967ec57e8..ab8eea05e 100644 --- a/commands/all/api/outbounds_remove.go +++ b/commands/all/api/outbounds_remove.go @@ -3,9 +3,10 @@ package api import ( "fmt" - handlerService "v2ray.com/core/app/proxyman/command" - "v2ray.com/core/commands/base" - "v2ray.com/core/infra/conf/serial" + handlerService "github.com/v2fly/v2ray-core/v4/app/proxyman/command" + "github.com/v2fly/v2ray-core/v4/commands/base" + "github.com/v2fly/v2ray-core/v4/common/cmdarg" + "github.com/v2fly/v2ray-core/v4/infra/conf/serial" ) var cmdRemoveOutbounds = &base.Command{ @@ -41,7 +42,7 @@ func executeRemoveOutbounds(cmd *base.Command, args []string) { tags := make([]string, 0) for _, arg := range unnamedArgs { - if r, err := loadArg(arg); err == nil { + if r, err := cmdarg.LoadArg(arg); err == nil { conf, err := serial.DecodeJSONConfig(r) if err != nil { base.Fatalf("failed to decode %s: %s", arg, err) diff --git a/commands/all/api/shared.go b/commands/all/api/shared.go index 830034046..5baadff03 100644 --- a/commands/all/api/shared.go +++ b/commands/all/api/shared.go @@ -1,21 +1,14 @@ package api import ( - "bytes" "context" "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" "strings" "time" + "github.com/v2fly/v2ray-core/v4/commands/base" "google.golang.org/grpc" "google.golang.org/protobuf/proto" - "v2ray.com/core/commands/base" - "v2ray.com/core/common/buf" ) type serviceHandler func(ctx context.Context, conn *grpc.ClientConn, cmd *base.Command, args []string) string @@ -45,63 +38,6 @@ func dialAPIServer() (conn *grpc.ClientConn, ctx context.Context, close func()) return } -// loadArg loads one arg, maybe an remote url, or local file path -func loadArg(arg string) (out io.Reader, err error) { - var data []byte - switch { - case strings.HasPrefix(arg, "http://"), strings.HasPrefix(arg, "https://"): - data, err = fetchHTTPContent(arg) - - case arg == "stdin:": - data, err = ioutil.ReadAll(os.Stdin) - - default: - data, err = ioutil.ReadFile(arg) - } - - if err != nil { - return - } - out = bytes.NewBuffer(data) - return -} - -// fetchHTTPContent dials https for remote content -func fetchHTTPContent(target string) ([]byte, error) { - parsedTarget, err := url.Parse(target) - if err != nil { - return nil, err - } - - if s := strings.ToLower(parsedTarget.Scheme); s != "http" && s != "https" { - return nil, fmt.Errorf("invalid scheme: %s", parsedTarget.Scheme) - } - - client := &http.Client{ - Timeout: 30 * time.Second, - } - resp, err := client.Do(&http.Request{ - Method: "GET", - URL: parsedTarget, - Close: true, - }) - if err != nil { - return nil, fmt.Errorf("failed to dial to %s", target) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("unexpected HTTP status code: %d", resp.StatusCode) - } - - content, err := buf.ReadAllToBytes(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read HTTP response") - } - - return content, nil -} - func showResponese(m proto.Message) { msg := "" bs, err := proto.Marshal(m) diff --git a/commands/all/api/stats_get.go b/commands/all/api/stats_get.go index 7e9b69ed0..22c96de84 100644 --- a/commands/all/api/stats_get.go +++ b/commands/all/api/stats_get.go @@ -1,8 +1,8 @@ package api import ( - statsService "v2ray.com/core/app/stats/command" - "v2ray.com/core/commands/base" + statsService "github.com/v2fly/v2ray-core/v4/app/stats/command" + "github.com/v2fly/v2ray-core/v4/commands/base" ) var cmdGetStats = &base.Command{ diff --git a/commands/all/api/stats_query.go b/commands/all/api/stats_query.go index 65670d399..ef7056a64 100644 --- a/commands/all/api/stats_query.go +++ b/commands/all/api/stats_query.go @@ -1,8 +1,8 @@ package api import ( - statsService "v2ray.com/core/app/stats/command" - "v2ray.com/core/commands/base" + statsService "github.com/v2fly/v2ray-core/v4/app/stats/command" + "github.com/v2fly/v2ray-core/v4/commands/base" ) var cmdQueryStats = &base.Command{ diff --git a/commands/all/api/stats_sys.go b/commands/all/api/stats_sys.go index 257372d39..58ef1382e 100644 --- a/commands/all/api/stats_sys.go +++ b/commands/all/api/stats_sys.go @@ -1,8 +1,8 @@ package api import ( - statsService "v2ray.com/core/app/stats/command" - "v2ray.com/core/commands/base" + statsService "github.com/v2fly/v2ray-core/v4/app/stats/command" + "github.com/v2fly/v2ray-core/v4/commands/base" ) var cmdSysStats = &base.Command{ diff --git a/commands/all/commands.go b/commands/all/commands.go index 1b533b79a..218875456 100644 --- a/commands/all/commands.go +++ b/commands/all/commands.go @@ -17,7 +17,6 @@ func init() { tls.CmdTLS, cmdUUID, cmdVerify, - cmdMerge, // documents docFormat, diff --git a/commands/all/convert.go b/commands/all/convert.go index ce1fc275f..4c16d925e 100644 --- a/commands/all/convert.go +++ b/commands/all/convert.go @@ -2,33 +2,54 @@ package all import ( "bytes" + "encoding/json" + "google.golang.org/protobuf/proto" "os" + "strings" "github.com/v2fly/v2ray-core/v4/commands/base" - "github.com/v2fly/v2ray-core/v4/infra/conf/merge" "github.com/v2fly/v2ray-core/v4/infra/conf/serial" - "google.golang.org/protobuf/proto" + "gopkg.in/yaml.v2" ) var cmdConvert = &base.Command{ - UsageLine: "{{.Exec}} convert [-r] [c1.json] [.json] [dir1] ...", - Short: "Convert multiple json config to protobuf", + CustomFlags: true, + UsageLine: "{{.Exec}} convert [c1.json] [.json] [dir1] ...", + Short: "Convert config files", Long: ` -Convert JSON config to protobuf. - -If multiple JSON files or folders specified, it merges them first, then convert. +Convert config files between different formats. Files are merged +before convert if multiple assigned. Arguments: + -i, -input + Specify the input format. + Available values: "json", "yaml" + Default: "json" + + -o, -output + Specify the output format + Available values: "json", "yaml", "protobuf" / "pb" + Default: "json" + -r Load confdir recursively. Examples: - {{.Exec}} {{.LongName}} config.json - {{.Exec}} {{.LongName}} c1.json c2.json - {{.Exec}} {{.LongName}} c1.json https://url.to/c2.json - {{.Exec}} {{.LongName}} "path/to/json_dir" + {{.Exec}} {{.LongName}} -output=protobuf config.json (1) + {{.Exec}} {{.LongName}} -output=yaml config.json (2) + {{.Exec}} {{.LongName}} -input=yaml config.yaml (3) + {{.Exec}} {{.LongName}} "path/to/dir" (4) + {{.Exec}} {{.LongName}} -i yaml -o protobuf c1.yaml .yaml (5) + +(1) Convert json to protobuf +(2) Convert json to yaml +(3) Convert yaml to json +(4) Merge json files in dir +(5) Merge yaml files and convert to protobuf + +Use "{{.Exec}} help config-merge" for more information about merge. `, } @@ -36,36 +57,76 @@ func init() { cmdConvert.Run = executeConvert // break init loop } -var convertReadDirRecursively = cmdConvert.Flag.Bool("r", false, "") +var ( + inputFormat string + outputFormat string + confDirRecursively bool +) +var formatExtensions = map[string][]string{ + "json": {".json", ".jsonc"}, + "yaml": {".yaml", ".yml"}, +} +func setConfArgs(cmd *base.Command) { + cmd.Flag.StringVar(&inputFormat, "input", "json", "") + cmd.Flag.StringVar(&inputFormat, "i", "json", "") + cmd.Flag.StringVar(&outputFormat, "output", "json", "") + cmd.Flag.StringVar(&outputFormat, "o", "json", "") + cmd.Flag.BoolVar(&confDirRecursively, "r", true, "") +} func executeConvert(cmd *base.Command, args []string) { + setConfArgs(cmd) + cmd.Flag.Parse(args) unnamed := cmd.Flag.Args() - files := resolveFolderToFiles(unnamed, *convertReadDirRecursively) + inputFormat = strings.ToLower(inputFormat) + outputFormat = strings.ToLower(outputFormat) + + files := resolveFolderToFiles(unnamed, formatExtensions[inputFormat], confDirRecursively) if len(files) == 0 { base.Fatalf("empty config list") } + m := mergeConvertToMap(files, inputFormat) - data, err := merge.FilesToJSON(files) - if err != nil { - base.Fatalf("failed to load json: %s", err) - } - r := bytes.NewReader(data) - cf, err := serial.DecodeJSONConfig(r) - if err != nil { - base.Fatalf("failed to decode json: %s", err) + var ( + out []byte + err error + ) + switch outputFormat { + case "json": + out, err = json.Marshal(m) + if err != nil { + base.Fatalf("failed to marshal json: %s", err) + } + case "yaml": + out, err = yaml.Marshal(m) + if err != nil { + base.Fatalf("failed to marshal json: %s", err) + } + case "pb", "protobuf": + data, err := json.Marshal(m) + if err != nil { + base.Fatalf("failed to marshal json: %s", err) + } + r := bytes.NewReader(data) + cf, err := serial.DecodeJSONConfig(r) + if err != nil { + base.Fatalf("failed to decode json: %s", err) + } + pbConfig, err := cf.Build() + if err != nil { + base.Fatalf(err.Error()) + } + out, err = proto.Marshal(pbConfig) + if err != nil { + base.Fatalf("failed to marshal proto config: %s", err) + } + default: + base.Errorf("invalid output format: %s", outputFormat) + base.Errorf("Run '%s help %s' for details.", base.CommandEnv.Exec, cmd.LongName()) + base.Exit() } - pbConfig, err := cf.Build() - if err != nil { - base.Fatalf(err.Error()) - } - - bytesConfig, err := proto.Marshal(pbConfig) - if err != nil { - base.Fatalf("failed to marshal proto config: %s", err) - } - - if _, err := os.Stdout.Write(bytesConfig); err != nil { + if _, err := os.Stdout.Write(out); err != nil { base.Fatalf("failed to write proto config: %s", err) } } diff --git a/commands/all/convert_confs.go b/commands/all/convert_confs.go new file mode 100644 index 000000000..8584c9948 --- /dev/null +++ b/commands/all/convert_confs.go @@ -0,0 +1,113 @@ +package all + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/v2fly/v2ray-core/v4/commands/base" + "github.com/v2fly/v2ray-core/v4/common/cmdarg" + "github.com/v2fly/v2ray-core/v4/infra/conf/json" + "github.com/v2fly/v2ray-core/v4/infra/conf/merge" +) + +func mergeConvertToMap(files []string, format string) map[string]interface{} { + var ( + m map[string]interface{} + err error + ) + switch inputFormat { + case "json": + m, err = merge.FilesToMap(files) + if err != nil { + base.Fatalf("failed to load json: %s", err) + } + case "yaml": + bs, err := yamlsToJSONs(files) + if err != nil { + base.Fatalf("failed to convert yaml to json: %s", err) + } + m, err = merge.BytesToMap(bs) + if err != nil { + base.Fatalf("failed to merge converted json: %s", err) + } + default: + base.Errorf("invalid input format: %s", format) + base.Errorf("Run '%s help %s' for details.", base.CommandEnv.Exec, cmdConvert.LongName()) + base.Exit() + } + return m +} + +// resolveFolderToFiles expands folder path (if any and it exists) to file paths. +// Any other paths, like file, even URL, it returns them as is. +func resolveFolderToFiles(paths []string, extensions []string, recursively bool) []string { + dirReader := readConfDir + if recursively { + dirReader = readConfDirRecursively + } + files := make([]string, 0) + for _, p := range paths { + i, err := os.Stat(p) + if err == nil && i.IsDir() { + files = append(files, dirReader(p, extensions)...) + continue + } + files = append(files, p) + } + return files +} + +func readConfDir(dirPath string, extensions []string) []string { + confs, err := ioutil.ReadDir(dirPath) + if err != nil { + base.Fatalf("failed to read dir %s: %s", dirPath, err) + } + files := make([]string, 0) + for _, f := range confs { + ext := filepath.Ext(f.Name()) + for _, e := range extensions { + if strings.EqualFold(ext, e) { + files = append(files, filepath.Join(dirPath, f.Name())) + break + } + } + } + return files +} + +// getFolderFiles get files in the folder and it's children +func readConfDirRecursively(dirPath string, extensions []string) []string { + files := make([]string, 0) + err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + ext := filepath.Ext(path) + for _, e := range extensions { + if strings.EqualFold(ext, e) { + files = append(files, path) + break + } + } + return nil + }) + if err != nil { + base.Fatalf("failed to read dir %s: %s", dirPath, err) + } + return files +} + +func yamlsToJSONs(files []string) ([][]byte, error) { + jsons := make([][]byte, 0) + for _, file := range files { + bs, err := cmdarg.LoadArgToBytes(file) + if err != nil { + return nil, err + } + j, err := json.FromYAML(bs) + if err != nil { + return nil, err + } + jsons = append(jsons, j) + } + return jsons, nil +} diff --git a/commands/all/format_doc.go b/commands/all/format_doc.go index ec93d2325..70bc7c1dd 100644 --- a/commands/all/format_doc.go +++ b/commands/all/format_doc.go @@ -14,7 +14,7 @@ var docFormat = &base.Command{ The default loader, multiple config files support. * yaml (.yml) - The yaml loader (coming soon?), multiple config files support. + The yaml loader, multiple config files support. * protobuf / pb (.pb) Single conifg file support. If multiple files assigned, diff --git a/commands/all/merge.go b/commands/all/merge.go deleted file mode 100644 index e484594ed..000000000 --- a/commands/all/merge.go +++ /dev/null @@ -1,101 +0,0 @@ -package all - -import ( - "io/ioutil" - "os" - "path/filepath" - - "github.com/v2fly/v2ray-core/v4/commands/base" - "github.com/v2fly/v2ray-core/v4/infra/conf/merge" -) - -var cmdMerge = &base.Command{ - UsageLine: "{{.Exec}} merge [-r] [c1.json] [url] [dir1] ...", - Short: "Merge json files into one", - Long: ` -Merge JSON files into one. - -Arguments: - - -r - Load confdir recursively. - -Examples: - - {{.Exec}} {{.LongName}} c1.json c2.json - {{.Exec}} {{.LongName}} c1.json https://url.to/c2.json - {{.Exec}} {{.LongName}} "path/to/json_dir" -`, -} - -func init() { - cmdMerge.Run = executeMerge -} - -var mergeReadDirRecursively = cmdMerge.Flag.Bool("r", false, "") - -func executeMerge(cmd *base.Command, args []string) { - unnamed := cmd.Flag.Args() - files := resolveFolderToFiles(unnamed, *mergeReadDirRecursively) - if len(files) == 0 { - base.Fatalf("empty config list") - } - - data, err := merge.FilesToJSON(files) - if err != nil { - base.Fatalf(err.Error()) - } - if _, err := os.Stdout.Write(data); err != nil { - base.Fatalf(err.Error()) - } -} - -// resolveFolderToFiles expands folder path (if any and it exists) to file paths. -// Any other paths, like file, even URL, it returns them as is. -func resolveFolderToFiles(paths []string, recursively bool) []string { - dirReader := readConfDir - if recursively { - dirReader = readConfDirRecursively - } - files := make([]string, 0) - for _, p := range paths { - i, err := os.Stat(p) - if err == nil && i.IsDir() { - files = append(files, dirReader(p)...) - continue - } - files = append(files, p) - } - return files -} - -func readConfDir(dirPath string) []string { - confs, err := ioutil.ReadDir(dirPath) - if err != nil { - base.Fatalf("failed to read dir %s: %s", dirPath, err) - } - files := make([]string, 0) - for _, f := range confs { - ext := filepath.Ext(f.Name()) - if ext == ".json" || ext == ".jsonc" { - files = append(files, filepath.Join(dirPath, f.Name())) - } - } - return files -} - -// getFolderFiles get files in the folder and it's children -func readConfDirRecursively(dirPath string) []string { - files := make([]string, 0) - err := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { - ext := filepath.Ext(path) - if ext == ".json" || ext == ".jsonc" { - files = append(files, path) - } - return nil - }) - if err != nil { - base.Fatalf("failed to read dir %s: %s", dirPath, err) - } - return files -} diff --git a/commands/all/merge_doc.go b/commands/all/merge_doc.go index 1eac1ca48..4ada22921 100644 --- a/commands/all/merge_doc.go +++ b/commands/all/merge_doc.go @@ -5,15 +5,18 @@ import ( ) var docMerge = &base.Command{ - UsageLine: "{{.Exec}} json-merge", - Short: "json merge logic", + UsageLine: "{{.Exec}} config-merge", + Short: "config merge logic", Long: ` -Merging of JSON configs is applied in following commands: +Merging of config files is applied in following commands: {{.Exec}} run -c c1.json -c c2.json ... - {{.Exec}} merge c1.json https://url.to/c2.json ... + {{.Exec}} test -c c1.yaml -c c2.yaml ... {{.Exec}} convert c1.json dir1 ... +Support of yaml is implemented by converting yaml to json, +both merge and load. So we take json as example here. + Suppose we have 2 JSON files, The 1st one: diff --git a/commands/all/tls/tls.go b/commands/all/tls/tls.go index a9dec0c34..4df0e79f3 100644 --- a/commands/all/tls/tls.go +++ b/commands/all/tls/tls.go @@ -1,7 +1,7 @@ package tls import ( - "v2ray.com/core/commands/base" + "github.com/v2fly/v2ray-core/v4/commands/base" ) // CmdTLS holds all tls sub commands diff --git a/infra/conf/merge/file.go b/common/cmdarg/arg.go similarity index 65% rename from infra/conf/merge/file.go rename to common/cmdarg/arg.go index a1d094213..79b625058 100644 --- a/infra/conf/merge/file.go +++ b/common/cmdarg/arg.go @@ -1,4 +1,4 @@ -package merge +package cmdarg import ( "bytes" @@ -11,29 +11,36 @@ import ( "time" "github.com/v2fly/v2ray-core/v4/common/buf" - "github.com/v2fly/v2ray-core/v4/infra/conf/serial" ) -// loadArg loads one arg, maybe an remote url, or local file path -func loadArg(arg string) (out io.Reader, err error) { - var data []byte +// LoadArg loads one arg, maybe an remote url, or local file path +func LoadArg(arg string) (out io.Reader, err error) { + bs, err := LoadArgToBytes(arg) + if err != nil { + return nil, err + } + out = bytes.NewBuffer(bs) + return +} + +// LoadArgToBytes loads one arg to []byte, maybe an remote url, or local file path +func LoadArgToBytes(arg string) (out []byte, err error) { switch { case strings.HasPrefix(arg, "http://"), strings.HasPrefix(arg, "https://"): - data, err = fetchHTTPContent(arg) + out, err = FetchHTTPContent(arg) case (arg == "stdin:"): - data, err = ioutil.ReadAll(os.Stdin) + out, err = ioutil.ReadAll(os.Stdin) default: - data, err = ioutil.ReadFile(arg) + out, err = ioutil.ReadFile(arg) } if err != nil { return } - out = bytes.NewBuffer(data) return } -// fetchHTTPContent dials https for remote content -func fetchHTTPContent(target string) ([]byte, error) { +// FetchHTTPContent dials https for remote content +func FetchHTTPContent(target string) ([]byte, error) { parsedTarget, err := url.Parse(target) if err != nil { return nil, newError("invalid URL: ", target).Base(err) @@ -67,12 +74,3 @@ func fetchHTTPContent(target string) ([]byte, error) { return content, nil } - -func decode(r io.Reader) (map[string]interface{}, error) { - c := make(map[string]interface{}) - err := serial.DecodeJSON(r, &c) - if err != nil { - return nil, err - } - return c, nil -} diff --git a/main/jsonem/errors.generated.go b/common/cmdarg/errors.generated.go similarity index 93% rename from main/jsonem/errors.generated.go rename to common/cmdarg/errors.generated.go index aaafcde6b..098f98f46 100644 --- a/main/jsonem/errors.generated.go +++ b/common/cmdarg/errors.generated.go @@ -1,4 +1,4 @@ -package jsonem +package cmdarg import "github.com/v2fly/v2ray-core/v4/common/errors" diff --git a/config.go b/config.go index a176ffe13..75d19dfce 100644 --- a/config.go +++ b/config.go @@ -13,7 +13,6 @@ import ( "github.com/v2fly/v2ray-core/v4/common" "github.com/v2fly/v2ray-core/v4/common/buf" "github.com/v2fly/v2ray-core/v4/common/cmdarg" - "github.com/v2fly/v2ray-core/v4/main/confloader" ) // ConfigFormat is a configurable format of V2Ray config file. @@ -109,7 +108,7 @@ func init() { Loader: func(input interface{}) (*Config, error) { switch v := input.(type) { case cmdarg.Arg: - r, err := confloader.LoadConfig(v[0]) + r, err := cmdarg.LoadArg(v[0]) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index c9b887544..1fc270721 100644 --- a/go.mod +++ b/go.mod @@ -1,51 +1,28 @@ module github.com/v2fly/v2ray-core/v4 -go 1.17 +go 1.16 require ( github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 github.com/google/go-cmp v0.5.6 github.com/gorilla/websocket v1.4.2 - github.com/jhump/protoreflect v1.9.0 - github.com/lucas-clemente/quic-go v0.23.0 - github.com/miekg/dns v1.1.43 - github.com/pires/go-proxyproto v0.6.0 + github.com/jhump/protoreflect v1.8.2 + github.com/lucas-clemente/quic-go v0.21.1 + github.com/miekg/dns v1.1.42 + github.com/pires/go-proxyproto v0.5.0 github.com/seiflotfy/cuckoofilter v0.0.0-20201222105146-bc6005554a0c github.com/stretchr/testify v1.7.0 github.com/v2fly/BrowserBridge v0.0.0-20210430233438-0570fc1d7d08 github.com/v2fly/VSign v0.0.0-20201108000810-e2adc24bf848 github.com/v2fly/ss-bloomring v0.0.0-20210312155135-28617310f63e go.starlark.net v0.0.0-20210602144842-1cdb82c9e17a - golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 - golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d + golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a + golang.org/x/net v0.0.0-20210614182718-04defd469f4e golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210820121016-41cdb8703e55 - google.golang.org/grpc v1.40.0 - google.golang.org/protobuf v1.27.1 - h12.io/socks v1.0.3 -) - -require ( - github.com/cheekybits/genny v1.0.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgryski/go-metro v0.0.0-20200812162917-85c65e2d0165 // indirect - github.com/ebfe/bcrypt_pbkdf v0.0.0-20140212075826-3c8d2dcb253a // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect - github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 // indirect - github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect - github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect - github.com/nxadm/tail v1.4.8 // indirect - github.com/onsi/ginkgo v1.16.4 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/riobard/go-bloom v0.0.0-20200614022211-cdc8013cb5b3 // indirect - github.com/xtaci/smux v1.5.15 // indirect - golang.org/x/mod v0.4.2 // indirect - golang.org/x/text v0.3.6 // indirect - golang.org/x/tools v0.1.1 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect - gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c // indirect + golang.org/x/sys v0.0.0-20210611083646-a4fc73990273 + google.golang.org/grpc v1.38.0 + google.golang.org/protobuf v1.26.0 + gopkg.in/yaml.v2 v2.4.0 + h12.io/socks v1.0.2 ) diff --git a/infra/conf/json/yaml.go b/infra/conf/json/yaml.go new file mode 100644 index 000000000..55180573d --- /dev/null +++ b/infra/conf/json/yaml.go @@ -0,0 +1,49 @@ +package json + +import ( + "encoding/json" + "fmt" + + "gopkg.in/yaml.v2" +) + +// FromYAML convert yaml to json +func FromYAML(v []byte) ([]byte, error) { + m1 := make(map[interface{}]interface{}) + err := yaml.Unmarshal(v, &m1) + if err != nil { + return nil, err + } + m2 := convert(m1) + j, err := json.Marshal(m2) + if err != nil { + return nil, err + } + return j, nil +} + +func convert(m map[interface{}]interface{}) map[string]interface{} { + res := map[string]interface{}{} + for k, v := range m { + var value interface{} + switch v2 := v.(type) { + case map[interface{}]interface{}: + value = convert(v2) + case []interface{}: + for i, el := range v2 { + if m, ok := el.(map[interface{}]interface{}); ok { + v2[i] = convert(m) + } + } + value = v2 + default: + value = v + } + key := "null" + if k != nil { + key = fmt.Sprint(k) + } + res[key] = value + } + return res +} diff --git a/infra/conf/json/yaml_test.go b/infra/conf/json/yaml_test.go new file mode 100644 index 000000000..84ada0a86 --- /dev/null +++ b/infra/conf/json/yaml_test.go @@ -0,0 +1,145 @@ +package json + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestYMLToJSON_V2Style(t *testing.T) { + input := ` +log: + loglevel: debug +inbounds: +- port: 10800 + listen: 127.0.0.1 + protocol: socks + settings: + udp: true +outbounds: +- protocol: vmess + settings: + vnext: + - address: example.com + port: 443 + users: + - id: '98a15fa6-2eb1-edd5-50ea-cfc428aaab78' + streamSettings: + network: tcp + security: tls +` + expected := ` +{ + "log": { + "loglevel": "debug" + }, + "inbounds": [{ + "port": 10800, + "listen": "127.0.0.1", + "protocol": "socks", + "settings": { + "udp": true + } + }], + "outbounds": [{ + "protocol": "vmess", + "settings": { + "vnext": [{ + "port": 443, + "address": "example.com", + "users": [{ + "id": "98a15fa6-2eb1-edd5-50ea-cfc428aaab78" + }] + }] + }, + "streamSettings": { + "network": "tcp", + "security": "tls" + } + }] +} +` + bs, err := FromYAML([]byte(input)) + if err != nil { + t.Error(err) + } + m := make(map[string]interface{}) + json.Unmarshal(bs, &m) + assertResult(t, m, expected) +} +func TestYMLToJSON_ValueTypes(t *testing.T) { + input := ` +boolean: + - TRUE + - FALSE + - true + - false +float: + - 3.14 + - 6.8523015e+5 +int: + - 123 + - 0b1010_0111_0100_1010_1110 +null: + nodeName: 'node' + parent: ~ # ~ for null +string: + - 哈哈 + - 'Hello world' + - newline + newline2 # multi-line string +date: + - 2018-02-17 # yyyy-MM-dd +datetime: + - 2018-02-17T15:02:31+08:00 # ISO 8601 time +mixed: + - true + - false + - 1 + - 0 + - null + - hello +# arbitrary keys +1: 0 +true: false +TRUE: TRUE +"str": "hello" +` + expected := ` +{ + "boolean": [true, false, true, false], + "float": [3.14, 685230.15], + "int": [123, 685230], + "null": { + "nodeName": "node", + "parent": null + }, + "string": ["哈哈", "Hello world", "newline newline2"], + "date": ["2018-02-17"], + "datetime": ["2018-02-17T15:02:31+08:00"], + "mixed": [true,false,1,0,null,"hello"], + "1": 0, + "true": true, + "str": "hello" +} +` + bs, err := FromYAML([]byte(input)) + if err != nil { + t.Error(err) + } + m := make(map[string]interface{}) + json.Unmarshal(bs, &m) + assertResult(t, m, expected) +} + +func assertResult(t *testing.T, value map[string]interface{}, expected string) { + e := make(map[string]interface{}) + err := json.Unmarshal([]byte(expected), &e) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(value, e) { + bs, _ := json.Marshal(value) + t.Fatalf("expected:\n%s\n\nactual:\n%s", expected, string(bs)) + } +} diff --git a/infra/conf/merge/merge.go b/infra/conf/merge/merge.go index 3d2badd7b..a07ab44af 100644 --- a/infra/conf/merge/merge.go +++ b/infra/conf/merge/merge.go @@ -18,6 +18,10 @@ package merge import ( "bytes" "encoding/json" + "io" + + "github.com/v2fly/v2ray-core/v4/common/cmdarg" + "github.com/v2fly/v2ray-core/v4/infra/conf/serial" ) // FilesToJSON merges multiple jsons files into one json, accepts remote url, or local file path @@ -65,9 +69,9 @@ func BytesToMap(args [][]byte) (m map[string]interface{}, err error) { } func loadFiles(args []string) (map[string]interface{}, error) { - conf := make(map[string]interface{}) + c := make(map[string]interface{}) for _, arg := range args { - r, err := loadArg(arg) + r, err := cmdarg.LoadArg(arg) if err != nil { return nil, err } @@ -75,11 +79,11 @@ func loadFiles(args []string) (map[string]interface{}, error) { if err != nil { return nil, err } - if err = mergeMaps(conf, m); err != nil { + if err = mergeMaps(c, m); err != nil { return nil, err } } - return conf, nil + return c, nil } func loadBytes(args [][]byte) (map[string]interface{}, error) { @@ -96,3 +100,12 @@ func loadBytes(args [][]byte) (map[string]interface{}, error) { } return conf, nil } + +func decode(r io.Reader) (map[string]interface{}, error) { + c := make(map[string]interface{}) + err := serial.DecodeJSON(r, &c) + if err != nil { + return nil, err + } + return c, nil +} diff --git a/main/commands/run.go b/main/commands/run.go index ad46b2dcd..080b38f8e 100644 --- a/main/commands/run.go +++ b/main/commands/run.go @@ -56,7 +56,7 @@ var ( ) func setConfigFlags(cmd *base.Command) { - configFormat = cmd.Flag.String("format", "json", "") + configFormat = cmd.Flag.String("format", "", "") configDirRecursively = cmd.Flag.Bool("r", false, "") cmd.Flag.Var(&configFiles, "config", "") diff --git a/main/confloader/confloader.go b/main/confloader/confloader.go deleted file mode 100644 index 01cb92339..000000000 --- a/main/confloader/confloader.go +++ /dev/null @@ -1,36 +0,0 @@ -package confloader - -import ( - "io" - "os" -) - -type ( - configFileLoader func(string) (io.Reader, error) - extconfigLoader func([]string, io.Reader) (io.Reader, error) -) - -var ( - EffectiveConfigFileLoader configFileLoader - EffectiveExtConfigLoader extconfigLoader -) - -// LoadConfig reads from a path/url/stdin -// actual work is in external module -func LoadConfig(file string) (io.Reader, error) { - if EffectiveConfigFileLoader == nil { - newError("external config module not loaded, reading from stdin").AtInfo().WriteToLog() - return os.Stdin, nil - } - return EffectiveConfigFileLoader(file) -} - -// LoadExtConfig calls v2ctl to handle multiple config -// the actual work also in external module -func LoadExtConfig(files []string, reader io.Reader) (io.Reader, error) { - if EffectiveExtConfigLoader == nil { - return nil, newError("external config module not loaded").AtError() - } - - return EffectiveExtConfigLoader(files, reader) -} diff --git a/main/confloader/errors.generated.go b/main/confloader/errors.generated.go deleted file mode 100644 index faf13cf65..000000000 --- a/main/confloader/errors.generated.go +++ /dev/null @@ -1,9 +0,0 @@ -package confloader - -import "github.com/v2fly/v2ray-core/v4/common/errors" - -type errPathObjHolder struct{} - -func newError(values ...interface{}) *errors.Error { - return errors.New(values...).WithPathObj(errPathObjHolder{}) -} diff --git a/main/confloader/external/errors.generated.go b/main/confloader/external/errors.generated.go deleted file mode 100644 index 04cde6576..000000000 --- a/main/confloader/external/errors.generated.go +++ /dev/null @@ -1,9 +0,0 @@ -package external - -import "github.com/v2fly/v2ray-core/v4/common/errors" - -type errPathObjHolder struct{} - -func newError(values ...interface{}) *errors.Error { - return errors.New(values...).WithPathObj(errPathObjHolder{}) -} diff --git a/main/confloader/external/external.go b/main/confloader/external/external.go deleted file mode 100644 index 138ec66b3..000000000 --- a/main/confloader/external/external.go +++ /dev/null @@ -1,87 +0,0 @@ -package external - -//go:generate go run github.com/v2fly/v2ray-core/v4/common/errors/errorgen - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "strings" - "time" - - "github.com/v2fly/v2ray-core/v4/common/buf" - "github.com/v2fly/v2ray-core/v4/common/platform/ctlcmd" - "github.com/v2fly/v2ray-core/v4/main/confloader" -) - -func ConfigLoader(arg string) (out io.Reader, err error) { - var data []byte - switch { - case strings.HasPrefix(arg, "http://"), strings.HasPrefix(arg, "https://"): - data, err = FetchHTTPContent(arg) - - case arg == "stdin:": - data, err = ioutil.ReadAll(os.Stdin) - - default: - data, err = ioutil.ReadFile(arg) - } - - if err != nil { - return - } - out = bytes.NewBuffer(data) - return -} - -func FetchHTTPContent(target string) ([]byte, error) { - parsedTarget, err := url.Parse(target) - if err != nil { - return nil, newError("invalid URL: ", target).Base(err) - } - - if s := strings.ToLower(parsedTarget.Scheme); s != "http" && s != "https" { - return nil, newError("invalid scheme: ", parsedTarget.Scheme) - } - - client := &http.Client{ - Timeout: 30 * time.Second, - } - resp, err := client.Do(&http.Request{ - Method: "GET", - URL: parsedTarget, - Close: true, - }) - if err != nil { - return nil, newError("failed to dial to ", target).Base(err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return nil, newError("unexpected HTTP status code: ", resp.StatusCode) - } - - content, err := buf.ReadAllToBytes(resp.Body) - if err != nil { - return nil, newError("failed to read HTTP response").Base(err) - } - - return content, nil -} - -func ExtConfigLoader(files []string, reader io.Reader) (io.Reader, error) { - buf, err := ctlcmd.Run(append([]string{"convert"}, files...), reader) - if err != nil { - return nil, err - } - - return strings.NewReader(buf.String()), nil -} - -func init() { - confloader.EffectiveConfigFileLoader = ConfigLoader - confloader.EffectiveExtConfigLoader = ExtConfigLoader -} diff --git a/main/distro/all/all.go b/main/distro/all/all.go index 0cab1fe3e..a8ded332a 100644 --- a/main/distro/all/all.go +++ b/main/distro/all/all.go @@ -74,10 +74,10 @@ import ( // The following line loads JSON from v2ctl // _ "github.com/v2fly/v2ray-core/v4/main/json" // The following line loads JSON internally - _ "github.com/v2fly/v2ray-core/v4/main/jsonem" + _ "github.com/v2fly/v2ray-core/v4/main/json" - // Load config from file or http(s) - _ "github.com/v2fly/v2ray-core/v4/main/confloader/external" + // YAML config support. + _ "github.com/v2fly/v2ray-core/v4/main/yaml" // commands _ "github.com/v2fly/v2ray-core/v4/commands/all" diff --git a/main/json/config_json.go b/main/json/config_json.go deleted file mode 100644 index c78fbbf83..000000000 --- a/main/json/config_json.go +++ /dev/null @@ -1,38 +0,0 @@ -package json - -//go:generate go run github.com/v2fly/v2ray-core/v4/common/errors/errorgen - -import ( - "io" - "os" - - core "github.com/v2fly/v2ray-core/v4" - "github.com/v2fly/v2ray-core/v4/common" - "github.com/v2fly/v2ray-core/v4/common/cmdarg" - "github.com/v2fly/v2ray-core/v4/main/confloader" -) - -func init() { - common.Must(core.RegisterConfigLoader(&core.ConfigFormat{ - Name: []string{"JSON"}, - Extension: []string{".json", ".jsonc"}, - Loader: func(input interface{}) (*core.Config, error) { - switch v := input.(type) { - case cmdarg.Arg: - r, err := confloader.LoadExtConfig(v, os.Stdin) - if err != nil { - return nil, newError("failed to execute v2ctl to convert config file.").Base(err).AtWarning() - } - return core.LoadConfig("protobuf", "", r) - case io.Reader: - r, err := confloader.LoadExtConfig([]string{"stdin:"}, os.Stdin) - if err != nil { - return nil, newError("failed to execute v2ctl to convert config file.").Base(err).AtWarning() - } - return core.LoadConfig("protobuf", "", r) - default: - return nil, newError("unknown type") - } - }, - })) -} diff --git a/main/jsonem/jsonem.go b/main/json/json.go similarity index 98% rename from main/jsonem/jsonem.go rename to main/json/json.go index 252bb8e12..1e019ede1 100644 --- a/main/jsonem/jsonem.go +++ b/main/json/json.go @@ -1,4 +1,4 @@ -package jsonem +package json import ( "bytes" diff --git a/main/yaml/yaml.go b/main/yaml/yaml.go new file mode 100644 index 000000000..0b5ae2ad4 --- /dev/null +++ b/main/yaml/yaml.go @@ -0,0 +1,69 @@ +package yaml + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + + core "github.com/v2fly/v2ray-core/v4" + "github.com/v2fly/v2ray-core/v4/common" + "github.com/v2fly/v2ray-core/v4/common/cmdarg" + "github.com/v2fly/v2ray-core/v4/infra/conf/json" + "github.com/v2fly/v2ray-core/v4/infra/conf/merge" + "github.com/v2fly/v2ray-core/v4/infra/conf/serial" +) + +func init() { + common.Must(core.RegisterConfigLoader(&core.ConfigFormat{ + Name: []string{"YAML"}, + Extension: []string{".yml", ".yaml"}, + Loader: func(input interface{}) (*core.Config, error) { + switch v := input.(type) { + case cmdarg.Arg: + bs, err := yamlsToJSONs(v) + if err != nil { + return nil, err + } + data, err := merge.BytesToJSON(bs) + if err != nil { + return nil, err + } + r := bytes.NewReader(data) + cf, err := serial.DecodeJSONConfig(r) + if err != nil { + return nil, err + } + return cf.Build() + case io.Reader: + bs, err := ioutil.ReadAll(v) + if err != nil { + return nil, err + } + bs, err = json.FromYAML(bs) + if err != nil { + return nil, err + } + return serial.LoadJSONConfig(bytes.NewBuffer(bs)) + default: + return nil, errors.New("unknow type") + } + }, + })) +} + +func yamlsToJSONs(files []string) ([][]byte, error) { + jsons := make([][]byte, 0) + for _, file := range files { + bs, err := cmdarg.LoadArgToBytes(file) + if err != nil { + return nil, err + } + j, err := json.FromYAML(bs) + if err != nil { + return nil, err + } + jsons = append(jsons, j) + } + return jsons, nil +}