cmd/syncthing: Add cli as a subcommand (fixes #6566, fixes #4719) (#7364)

* cmd/syncthing: Add cli as a subcommand (fixes #6566, fixes #4719)

* Hijack help

* Add comment

* Revert go.mod/go.sum

cmd/syncthing/cli/client.go

@@ -0,0 +1,96 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/config"
+)
+
+type APIClient struct {
+	http.Client
+	cfg    config.GUIConfiguration
+	apikey string
+}
+
+func getClient(cfg config.GUIConfiguration) *APIClient {
+	httpClient := http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+			DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+				return net.Dial(cfg.Network(), cfg.Address())
+			},
+		},
+	}
+	return &APIClient{
+		Client: httpClient,
+		cfg:    cfg,
+		apikey: cfg.APIKey,
+	}
+}
+
+func (c *APIClient) Endpoint() string {
+	if c.cfg.Network() == "unix" {
+		return "http://unix/"
+	}
+	url := c.cfg.URL()
+	if !strings.HasSuffix(url, "/") {
+		url += "/"
+	}
+	return url
+}
+
+func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
+	req.Header.Set("X-API-Key", c.apikey)
+	resp, err := c.Client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, checkResponse(resp)
+}
+
+func (c *APIClient) Get(url string) (*http.Response, error) {
+	request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func (c *APIClient) Post(url, body string) (*http.Response, error) {
+	request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
+	if err != nil {
+		return nil, err
+	}
+	return c.Do(request)
+}
+
+func checkResponse(response *http.Response) error {
+	if response.StatusCode == 404 {
+		return errors.New("invalid endpoint or API call")
+	} else if response.StatusCode == 403 {
+		return errors.New("invalid API key")
+	} else if response.StatusCode != 200 {
+		data, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := strings.TrimSpace(string(data))
+		return fmt.Errorf("unexpected HTTP status returned: %s\n%s", response.Status, body)
+	}
+	return nil
+}

cmd/syncthing/cli/errors.go

@@ -0,0 +1,61 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/urfave/cli"
+)
+
+var errorsCommand = cli.Command{
+	Name:     "errors",
+	HideHelp: true,
+	Usage:    "Error command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "show",
+			Usage:  "Show pending errors",
+			Action: expects(0, dumpOutput("system/error")),
+		},
+		{
+			Name:      "push",
+			Usage:     "Push an error to active clients",
+			ArgsUsage: "[error message]",
+			Action:    expects(1, errorsPush),
+		},
+		{
+			Name:   "clear",
+			Usage:  "Clear pending errors",
+			Action: expects(0, emptyPost("system/error/clear")),
+		},
+	},
+}
+
+func errorsPush(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	errStr := strings.Join(c.Args(), " ")
+	response, err := client.Post("system/error", strings.TrimSpace(errStr))
+	if err != nil {
+		return err
+	}
+	if response.StatusCode != 200 {
+		errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
+		bytes, err := responseToBArray(response)
+		if err != nil {
+			return err
+		}
+		body := string(bytes)
+		if body != "" {
+			errStr += "\nBody: " + body
+		}
+		return errors.New(errStr)
+	}
+	return nil
+}

cmd/syncthing/cli/main.go

@@ -0,0 +1,208 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"bufio"
+	"crypto/tls"
+	"encoding/json"
+	"log"
+	"os"
+	"reflect"
+
+	"github.com/AudriusButkevicius/recli"
+	"github.com/flynn-archive/go-shlex"
+	"github.com/mattn/go-isatty"
+	"github.com/pkg/errors"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/locations"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/svcutil"
+
+	"github.com/urfave/cli"
+)
+
+type CLI struct {
+	GUIAddress string   `name:"gui-address" placeholder:"URL" help:"Override GUI address (e.g. \"http://192.0.2.42:8443\")"`
+	GUIAPIKey  string   `name:"gui-apikey" placeholder:"API-KEY" help:"Override GUI API key"`
+	HomeDir    string   `name:"home" placeholder:"PATH" help:"Set configuration and data directory"`
+	ConfDir    string   `name:"conf" placeholder:"PATH" help:"Set configuration directory (config and keys)"`
+	Args       []string `arg:"" optional:""`
+}
+
+func (c *CLI) Run() error {
+	// Not set as default above because the strings can be really long.
+	var err error
+	homeSet := c.HomeDir != ""
+	confSet := c.ConfDir != ""
+	switch {
+	case homeSet && confSet:
+		err = errors.New("-home must not be used together with -conf")
+	case homeSet:
+		err = locations.SetBaseDir(locations.ConfigBaseDir, c.HomeDir)
+	case confSet:
+		err = locations.SetBaseDir(locations.ConfigBaseDir, c.ConfDir)
+	}
+	if err != nil {
+		log.Println("Command line options:", err)
+		os.Exit(svcutil.ExitError.AsInt())
+	}
+	guiCfg := config.GUIConfiguration{
+		RawAddress: c.GUIAddress,
+		APIKey:     c.GUIAPIKey,
+	}
+
+	// Now if the API key and address is not provided (we are not connecting to a remote instance),
+	// try to rip it out of the config.
+	if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
+		// Load the certs and get the ID
+		cert, err := tls.LoadX509KeyPair(
+			locations.Get(locations.CertFile),
+			locations.Get(locations.KeyFile),
+		)
+		if err != nil {
+			return errors.Wrap(err, "reading device ID")
+		}
+
+		myID := protocol.NewDeviceID(cert.Certificate[0])
+
+		// Load the config
+		cfg, _, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
+		if err != nil {
+			return errors.Wrap(err, "loading config")
+		}
+
+		guiCfg = cfg.GUI()
+	} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
+		return errors.New("Both --gui-address and --gui-apikey should be specified")
+	}
+
+	if guiCfg.Address() == "" {
+		return errors.New("Could not find GUI Address")
+	}
+
+	if guiCfg.APIKey == "" {
+		return errors.New("Could not find GUI API key")
+	}
+
+	client := getClient(guiCfg)
+
+	cfg, err := getConfig(client)
+	original := cfg.Copy()
+	if err != nil {
+		return errors.Wrap(err, "getting config")
+	}
+
+	// Copy the config and set the default flags
+	recliCfg := recli.DefaultConfig
+	recliCfg.IDTag.Name = "xml"
+	recliCfg.SkipTag.Name = "json"
+
+	commands, err := recli.New(recliCfg).Construct(&cfg)
+	if err != nil {
+		return errors.Wrap(err, "config reflect")
+	}
+
+	// Implement the same flags at the upper CLI, but do nothing with them.
+	// This is so that the usage text is the same
+	fakeFlags := []cli.Flag{
+		cli.StringFlag{
+			Name:  "gui-address",
+			Value: "URL",
+			Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
+		},
+		cli.StringFlag{
+			Name:  "gui-apikey",
+			Value: "API-KEY",
+			Usage: "Override GUI API key",
+		},
+		cli.StringFlag{
+			Name:  "home",
+			Value: "PATH",
+			Usage: "Set configuration and data directory",
+		},
+		cli.StringFlag{
+			Name:  "conf",
+			Value: "PATH",
+			Usage: "Set configuration directory (config and keys)",
+		},
+	}
+
+	// Construct the actual CLI
+	app := cli.NewApp()
+	app.Name = "syncthing cli"
+	app.HelpName = app.Name
+	app.Author = "The Syncthing Authors"
+	app.Usage = "Syncthing command line interface"
+	app.Flags = fakeFlags
+	app.Metadata = map[string]interface{}{
+		"client": client,
+	}
+	app.Commands = []cli.Command{
+		{
+			Name:        "config",
+			HideHelp:    true,
+			Usage:       "Configuration modification command group",
+			Subcommands: commands,
+		},
+		showCommand,
+		operationCommand,
+		errorsCommand,
+	}
+
+	// It expects to be give os.Args which has argv[0] set to executable name, so fake it.
+	c.Args = append([]string{"cli"}, c.Args...)
+
+	tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
+	if !tty {
+		// Not a TTY, consume from stdin
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			input, err := shlex.Split(scanner.Text())
+			if err != nil {
+				return errors.Wrap(err, "parsing input")
+			}
+			if len(input) == 0 {
+				continue
+			}
+			err = app.Run(append(c.Args, input...))
+			if err != nil {
+				return err
+			}
+		}
+		err = scanner.Err()
+		if err != nil {
+			return err
+		}
+	} else {
+		err = app.Run(c.Args)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !reflect.DeepEqual(cfg, original) {
+		body, err := json.MarshalIndent(cfg, "", "  ")
+		if err != nil {
+			return err
+		}
+		resp, err := client.Post("system/config", string(body))
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode != 200 {
+			body, err := responseToBArray(resp)
+			if err != nil {
+				return err
+			}
+			return errors.New(string(body))
+		}
+	}
+	return nil
+}

cmd/syncthing/cli/operations.go

@@ -0,0 +1,78 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli"
+)
+
+var operationCommand = cli.Command{
+	Name:     "operations",
+	HideHelp: true,
+	Usage:    "Operation command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "restart",
+			Usage:  "Restart syncthing",
+			Action: expects(0, emptyPost("system/restart")),
+		},
+		{
+			Name:   "shutdown",
+			Usage:  "Shutdown syncthing",
+			Action: expects(0, emptyPost("system/shutdown")),
+		},
+		{
+			Name:   "reset",
+			Usage:  "Reset syncthing deleting all folders and devices",
+			Action: expects(0, emptyPost("system/reset")),
+		},
+		{
+			Name:   "upgrade",
+			Usage:  "Upgrade syncthing (if a newer version is available)",
+			Action: expects(0, emptyPost("system/upgrade")),
+		},
+		{
+			Name:      "folder-override",
+			Usage:     "Override changes on folder (remote for sendonly, local for receiveonly)",
+			ArgsUsage: "[folder id]",
+			Action:    expects(1, foldersOverride),
+		},
+	},
+}
+
+func foldersOverride(c *cli.Context) error {
+	client := c.App.Metadata["client"].(*APIClient)
+	cfg, err := getConfig(client)
+	if err != nil {
+		return err
+	}
+	rid := c.Args()[0]
+	for _, folder := range cfg.Folders {
+		if folder.ID == rid {
+			response, err := client.Post("db/override", "")
+			if err != nil {
+				return err
+			}
+			if response.StatusCode != 200 {
+				errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
+				bytes, err := responseToBArray(response)
+				if err != nil {
+					return err
+				}
+				body := string(bytes)
+				if body != "" {
+					errStr += "\nBody: " + body
+				}
+				return fmt.Errorf(errStr)
+			}
+			return nil
+		}
+	}
+	return fmt.Errorf("Folder " + rid + " not found")
+}

cmd/syncthing/cli/show.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"github.com/urfave/cli"
+)
+
+var showCommand = cli.Command{
+	Name:     "show",
+	HideHelp: true,
+	Usage:    "Show command group",
+	Subcommands: []cli.Command{
+		{
+			Name:   "version",
+			Usage:  "Show syncthing client version",
+			Action: expects(0, dumpOutput("system/version")),
+		},
+		{
+			Name:   "config-status",
+			Usage:  "Show configuration status, whether or not a restart is required for changes to take effect",
+			Action: expects(0, dumpOutput("system/config/insync")),
+		},
+		{
+			Name:   "system",
+			Usage:  "Show system status",
+			Action: expects(0, dumpOutput("system/status")),
+		},
+		{
+			Name:   "connections",
+			Usage:  "Report about connections to other devices",
+			Action: expects(0, dumpOutput("system/connections")),
+		},
+		{
+			Name:   "usage",
+			Usage:  "Show usage report",
+			Action: expects(0, dumpOutput("svc/report")),
+		},
+	},
+}

cmd/syncthing/cli/utils.go

@@ -0,0 +1,94 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/urfave/cli"
+)
+
+func responseToBArray(response *http.Response) ([]byte, error) {
+	bytes, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		return nil, err
+	}
+	return bytes, response.Body.Close()
+}
+
+func emptyPost(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		_, err := client.Post(url, "")
+		return err
+	}
+}
+
+func dumpOutput(url string) cli.ActionFunc {
+	return func(c *cli.Context) error {
+		client := c.App.Metadata["client"].(*APIClient)
+		response, err := client.Get(url)
+		if err != nil {
+			return err
+		}
+		return prettyPrintResponse(c, response)
+	}
+}
+
+func getConfig(c *APIClient) (config.Configuration, error) {
+	cfg := config.Configuration{}
+	response, err := c.Get("system/config")
+	if err != nil {
+		return cfg, err
+	}
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return cfg, err
+	}
+	err = json.Unmarshal(bytes, &cfg)
+	if err == nil {
+		return cfg, err
+	}
+	return cfg, nil
+}
+
+func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		if ctx.NArg() != n {
+			plural := ""
+			if n != 1 {
+				plural = "s"
+			}
+			return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
+		}
+		return actionFunc(ctx)
+	}
+}
+
+func prettyPrintJSON(data interface{}) error {
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	return enc.Encode(data)
+}
+
+func prettyPrintResponse(c *cli.Context, response *http.Response) error {
+	bytes, err := responseToBArray(response)
+	if err != nil {
+		return err
+	}
+	var data interface{}
+	if err := json.Unmarshal(bytes, &data); err != nil {
+		return err
+	}
+	// TODO: Check flag for pretty print format
+	return prettyPrintJSON(data)
+}

cmd/syncthing/main.go

@@ -31,6 +31,9 @@ import (
 	"time"
 
 	"github.com/alecthomas/kong"
+	"github.com/thejerf/suture/v4"
+
+	"github.com/syncthing/syncthing/cmd/syncthing/cli"
 	"github.com/syncthing/syncthing/cmd/syncthing/decrypt"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
@@ -48,7 +51,6 @@ import (
 	"github.com/syncthing/syncthing/lib/upgrade"
 
 	"github.com/pkg/errors"
-	"github.com/thejerf/suture/v4"
 )
 
 const (
@@ -127,11 +129,12 @@ var (
 	errTooEarlyUpgrade      = fmt.Errorf("last upgrade happened less than %v ago, skipping", upgradeRetryInterval)
 )
 
-// The cli struct is the main entry point for the command line parser. The
+// The entrypoint struct is the main entry point for the command line parser. The
 // commands and options here are top level commands to syncthing.
-var cli struct {
+var entrypoint struct {
 	Serve   serveOptions `cmd:"" help:"Run Syncthing"`
 	Decrypt decrypt.CLI  `cmd:"" help:"Decrypt or verify an encrypted folder"`
+	Cli     cli.CLI      `cmd:"" help:"Command line interface for Syncthing"`
 }
 
 // serveOptions are the options for the `syncthing serve` command.
@@ -227,11 +230,11 @@ func main() {
 		args = append([]string{"serve"}, convertLegacyArgs(args)...)
 	}
 
-	cli.Serve.setDefaults()
+	entrypoint.Serve.setDefaults()
 
 	// Create a parser with an overridden help function to print our extra
 	// help info.
-	parser, err := kong.New(&cli, kong.Help(extraHelpPrinter))
+	parser, err := kong.New(&entrypoint, kong.Help(helpHandler))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -242,7 +245,11 @@ func main() {
 	parser.FatalIfErrorf(err)
 }
 
-func extraHelpPrinter(options kong.HelpOptions, ctx *kong.Context) error {
+func helpHandler(options kong.HelpOptions, ctx *kong.Context) error {
+	// If we're looking for CLI help, pass the arguments down to the CLI library to print it's own help.
+	if ctx.Command() == "cli" {
+		return ctx.Run()
+	}
 	if err := kong.DefaultHelpPrinter(options, ctx); err != nil {
 		return err
 	}
@@ -282,12 +289,12 @@ func (options serveOptions) Run() error {
 	case homeSet && dataSet:
 		err = errors.New("-home must not be used together with -conf and -data")
 	case homeSet:
-		if err = setLocation(locations.ConfigBaseDir, options.HomeDir); err == nil {
-			err = setLocation(locations.DataBaseDir, options.HomeDir)
+		if err = locations.SetBaseDir(locations.ConfigBaseDir, options.HomeDir); err == nil {
+			err = locations.SetBaseDir(locations.DataBaseDir, options.HomeDir)
 		}
 	case dataSet:
-		if err = setLocation(locations.ConfigBaseDir, options.ConfDir); err == nil {
-			err = setLocation(locations.DataBaseDir, options.DataDir)
+		if err = locations.SetBaseDir(locations.ConfigBaseDir, options.ConfDir); err == nil {
+			err = locations.SetBaseDir(locations.DataBaseDir, options.DataDir)
 		}
 	}
 	if err != nil {
@@ -1004,17 +1011,6 @@ func exitCodeForUpgrade(err error) int {
 	return svcutil.ExitError.AsInt()
 }
 
-func setLocation(enum locations.BaseDirEnum, loc string) error {
-	if !filepath.IsAbs(loc) {
-		var err error
-		loc, err = filepath.Abs(loc)
-		if err != nil {
-			return err
-		}
-	}
-	return locations.SetBaseDir(enum, loc)
-}
-
 // convertLegacyArgs returns the slice of arguments with single dash long
 // flags converted to double dash long flags.
 func convertLegacyArgs(args []string) []string {