diff --git a/compose/cli/docopt_command.py b/compose/cli/docopt_command.py index d0ba7f670..e56b37835 100644 --- a/compose/cli/docopt_command.py +++ b/compose/cli/docopt_command.py @@ -17,10 +17,16 @@ class DocoptDispatcher: self.command_class = command_class self.options = options + @classmethod + def get_command_and_options(cls, doc_entity, argv, options): + command_help = getdoc(doc_entity) + opt = docopt_full_help(command_help, argv, **options) + command = opt['COMMAND'] + return command_help, opt, command + def parse(self, argv): - command_help = getdoc(self.command_class) - options = docopt_full_help(command_help, argv, **self.options) - command = options['COMMAND'] + command_help, options, command = DocoptDispatcher.get_command_and_options( + self.command_class, argv, self.options) if command is None: raise SystemExit(command_help) diff --git a/compose/cli/main.py b/compose/cli/main.py index c0fe3bb43..0f63b9791 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -26,6 +26,7 @@ from ..config.serialize import serialize_config from ..config.types import VolumeSpec from ..const import IS_WINDOWS_PLATFORM from ..errors import StreamParseError +from ..metrics.decorator import metrics from ..progress_stream import StreamOutputError from ..project import get_image_digests from ..project import MissingDigests @@ -53,6 +54,8 @@ from .log_printer import LogPrinter from .utils import get_version_info from .utils import human_readable_file_size from .utils import yesno +from compose.metrics.client import MetricsCommand +from compose.metrics.client import Status if not IS_WINDOWS_PLATFORM: @@ -62,36 +65,77 @@ log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) -def main(): +def main(): # noqa: C901 signals.ignore_sigpipe() + command = None try: - command = dispatch() - command() + _, opts, command = DocoptDispatcher.get_command_and_options( + TopLevelCommand, + get_filtered_args(sys.argv[1:]), + {'options_first': True, 'version': get_version_info('compose')}) + except Exception: + pass + try: + command_func = dispatch() + command_func() except (KeyboardInterrupt, signals.ShutdownException): - log.error("Aborting.") - sys.exit(1) + exit_with_metrics(command, "Aborting.", status=Status.FAILURE) except (UserError, NoSuchService, ConfigurationError, ProjectError, OperationFailedError) as e: - log.error(e.msg) - sys.exit(1) + exit_with_metrics(command, e.msg, status=Status.FAILURE) except BuildError as e: reason = "" if e.reason: reason = " : " + e.reason - log.error("Service '{}' failed to build{}".format(e.service.name, reason)) - sys.exit(1) + exit_with_metrics(command, + "Service '{}' failed to build{}".format(e.service.name, reason), + status=Status.FAILURE) except StreamOutputError as e: - log.error(e) - sys.exit(1) + exit_with_metrics(command, e, status=Status.FAILURE) except NeedsBuildError as e: - log.error("Service '{}' needs to be built, but --no-build was passed.".format(e.service.name)) - sys.exit(1) + exit_with_metrics(command, + "Service '{}' needs to be built, but --no-build was passed.".format( + e.service.name), status=Status.FAILURE) except NoSuchCommand as e: commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) - log.error("No such command: %s\n\n%s", e.command, commands) - sys.exit(1) + exit_with_metrics(e.command, "No such command: {}\n\n{}".format(e.command, commands)) except (errors.ConnectionError, StreamParseError): - sys.exit(1) + exit_with_metrics(command, status=Status.FAILURE) + except SystemExit as e: + status = Status.SUCCESS + if len(sys.argv) > 1 and '--help' not in sys.argv: + status = Status.FAILURE + + if command and len(sys.argv) >= 3 and sys.argv[2] == '--help': + command = '--help ' + command + + if not command and len(sys.argv) >= 2 and sys.argv[1] == '--help': + command = '--help' + + msg = e.args[0] if len(e.args) else "" + code = 0 + if isinstance(e.code, int): + code = e.code + exit_with_metrics(command, log_msg=msg, status=status, + exit_code=code) + + +def get_filtered_args(args): + if args[0] in ('-h', '--help'): + return [] + if args[0] == '--version': + return ['version'] + + +def exit_with_metrics(command, log_msg=None, status=Status.SUCCESS, exit_code=1): + if log_msg: + if not exit_code: + log.info(log_msg) + else: + log.error(log_msg) + + MetricsCommand(command, status=status).send_metrics() + sys.exit(exit_code) def dispatch(): @@ -133,8 +177,10 @@ def setup_logging(): root_logger.addHandler(console_handler) root_logger.setLevel(logging.DEBUG) - # Disable requests logging + # Disable requests and docker-py logging + logging.getLogger("urllib3").propagate = False logging.getLogger("requests").propagate = False + logging.getLogger("docker").propagate = False def setup_parallel_logger(noansi): @@ -254,6 +300,7 @@ class TopLevelCommand: environment_file = self.toplevel_options.get('--env-file') return Environment.from_env_file(self.project_dir, environment_file) + @metrics() def build(self, options): """ Build or rebuild services. @@ -305,6 +352,7 @@ class TopLevelCommand: progress=options.get('--progress'), ) + @metrics() def config(self, options): """ Validate and view the Compose file. @@ -354,6 +402,7 @@ class TopLevelCommand: print(serialize_config(compose_config, image_digests, not options['--no-interpolate'])) + @metrics() def create(self, options): """ Creates containers for a service. @@ -382,6 +431,7 @@ class TopLevelCommand: do_build=build_action_from_opts(options), ) + @metrics() def down(self, options): """ Stops containers and removes containers, networks, volumes, and images @@ -450,6 +500,7 @@ class TopLevelCommand: print(formatter(event)) sys.stdout.flush() + @metrics("exec") def exec_command(self, options): """ Execute a command in a running container @@ -526,6 +577,7 @@ class TopLevelCommand: sys.exit(exit_code) @classmethod + @metrics() def help(cls, options): """ Get help on a command. @@ -539,6 +591,7 @@ class TopLevelCommand: print(getdoc(subject)) + @metrics() def images(self, options): """ List images used by the created containers. @@ -593,6 +646,7 @@ class TopLevelCommand: ]) print(Formatter.table(headers, rows)) + @metrics() def kill(self, options): """ Force stop service containers. @@ -607,6 +661,7 @@ class TopLevelCommand: self.project.kill(service_names=options['SERVICE'], signal=signal) + @metrics() def logs(self, options): """ View output from containers. @@ -643,6 +698,7 @@ class TopLevelCommand: event_stream=self.project.events(service_names=options['SERVICE']), keep_prefix=not options['--no-log-prefix']).run() + @metrics() def pause(self, options): """ Pause services. @@ -652,6 +708,7 @@ class TopLevelCommand: containers = self.project.pause(service_names=options['SERVICE']) exit_if(not containers, 'No containers to pause', 1) + @metrics() def port(self, options): """ Print the public port for a port binding. @@ -673,6 +730,7 @@ class TopLevelCommand: options['PRIVATE_PORT'], protocol=options.get('--protocol') or 'tcp') or '') + @metrics() def ps(self, options): """ List containers. @@ -729,6 +787,7 @@ class TopLevelCommand: ]) print(Formatter.table(headers, rows)) + @metrics() def pull(self, options): """ Pulls images for services defined in a Compose file, but does not start the containers. @@ -752,6 +811,7 @@ class TopLevelCommand: include_deps=options.get('--include-deps'), ) + @metrics() def push(self, options): """ Pushes images for services. @@ -766,6 +826,7 @@ class TopLevelCommand: ignore_push_failures=options.get('--ignore-push-failures') ) + @metrics() def rm(self, options): """ Removes stopped service containers. @@ -810,6 +871,7 @@ class TopLevelCommand: else: print("No stopped containers") + @metrics() def run(self, options): """ Run a one-off command on a service. @@ -870,6 +932,7 @@ class TopLevelCommand: self.toplevel_options, self.toplevel_environment ) + @metrics() def scale(self, options): """ Set number of containers to run for a service. @@ -898,6 +961,7 @@ class TopLevelCommand: for service_name, num in parse_scale_args(options['SERVICE=NUM']).items(): self.project.get_service(service_name).scale(num, timeout=timeout) + @metrics() def start(self, options): """ Start existing containers. @@ -907,6 +971,7 @@ class TopLevelCommand: containers = self.project.start(service_names=options['SERVICE']) exit_if(not containers, 'No containers to start', 1) + @metrics() def stop(self, options): """ Stop running containers without removing them. @@ -922,6 +987,7 @@ class TopLevelCommand: timeout = timeout_from_opts(options) self.project.stop(service_names=options['SERVICE'], timeout=timeout) + @metrics() def restart(self, options): """ Restart running containers. @@ -936,6 +1002,7 @@ class TopLevelCommand: containers = self.project.restart(service_names=options['SERVICE'], timeout=timeout) exit_if(not containers, 'No containers to restart', 1) + @metrics() def top(self, options): """ Display the running processes @@ -963,6 +1030,7 @@ class TopLevelCommand: print(container.name) print(Formatter.table(headers, rows)) + @metrics() def unpause(self, options): """ Unpause services. @@ -972,6 +1040,7 @@ class TopLevelCommand: containers = self.project.unpause(service_names=options['SERVICE']) exit_if(not containers, 'No containers to unpause', 1) + @metrics() def up(self, options): """ Builds, (re)creates, starts, and attaches to containers for a service. @@ -1122,6 +1191,7 @@ class TopLevelCommand: sys.exit(exit_code) @classmethod + @metrics() def version(cls, options): """ Show version information and quit. diff --git a/compose/metrics/__init__.py b/compose/metrics/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/compose/metrics/client.py b/compose/metrics/client.py new file mode 100644 index 000000000..cf4afda5d --- /dev/null +++ b/compose/metrics/client.py @@ -0,0 +1,55 @@ +import os +from enum import Enum + +import requests +from docker import ContextAPI +from docker.transport import UnixHTTPAdapter + +from compose.const import IS_WINDOWS_PLATFORM + + +class Status(Enum): + SUCCESS = "success" + FAILURE = "failure" + CANCELED = "canceled" + + +class MetricsSource: + CLI = "docker-compose" + + +if IS_WINDOWS_PLATFORM: + METRICS_SOCKET_FILE = 'http+unix://\\\\.\\pipe\\docker_cli' +else: + METRICS_SOCKET_FILE = 'http+unix:///var/run/metrics-docker-cli.sock' + + +class MetricsCommand(requests.Session): + """ + Representation of a command in the metrics. + """ + + def __init__(self, command, + context_type=None, status=Status.SUCCESS, + source=MetricsSource.CLI, uri=None): + super().__init__() + self.command = "compose " + command if command else "compose --help" + self.context = context_type or ContextAPI.get_current_context().context_type or 'moby' + self.source = source + self.status = status.value + self.uri = uri or os.environ.get("METRICS_SOCKET_FILE", METRICS_SOCKET_FILE) + self.mount("http+unix://", UnixHTTPAdapter(self.uri)) + + def send_metrics(self): + try: + return self.post("http+unix://localhost/", json=self.to_map(), timeout=.05) + except Exception as e: + return e + + def to_map(self): + return { + 'command': self.command, + 'context': self.context, + 'source': self.source, + 'status': self.status, + } diff --git a/compose/metrics/decorator.py b/compose/metrics/decorator.py new file mode 100644 index 000000000..3126e6941 --- /dev/null +++ b/compose/metrics/decorator.py @@ -0,0 +1,21 @@ +import functools + +from compose.metrics.client import MetricsCommand +from compose.metrics.client import Status + + +class metrics: + def __init__(self, command_name=None): + self.command_name = command_name + + def __call__(self, fn): + @functools.wraps(fn, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES) + def wrapper(*args, **kwargs): + if not self.command_name: + self.command_name = fn.__name__ + result = fn(*args, **kwargs) + MetricsCommand(self.command_name, status=Status.SUCCESS).send_metrics() + return result + return wrapper diff --git a/script/test/all b/script/test/all index 64a5a99f2..1626fed37 100755 --- a/script/test/all +++ b/script/test/all @@ -21,6 +21,7 @@ elif [ "$DOCKER_VERSIONS" == "all" ]; then DOCKER_VERSIONS=$($get_versions -n 2 recent) fi +DOCKER_VERSIONS=19.03.14 BUILD_NUMBER=${BUILD_NUMBER-$USER} PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py39} diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 2ff515739..d0441d35c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -58,13 +58,16 @@ COMPOSE_COMPATIBILITY_DICT = { } -def start_process(base_dir, options): +def start_process(base_dir, options, executable=None, env=None): + executable = executable or DOCKER_COMPOSE_EXECUTABLE proc = subprocess.Popen( - [DOCKER_COMPOSE_EXECUTABLE] + options, + [executable] + options, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=base_dir) + cwd=base_dir, + env=env, + ) print("Running process: %s" % proc.pid) return proc @@ -78,9 +81,10 @@ def wait_on_process(proc, returncode=0, stdin=None): return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) -def dispatch(base_dir, options, project_options=None, returncode=0, stdin=None): +def dispatch(base_dir, options, + project_options=None, returncode=0, stdin=None, executable=None, env=None): project_options = project_options or [] - proc = start_process(base_dir, project_options + options) + proc = start_process(base_dir, project_options + options, executable=executable, env=env) return wait_on_process(proc, returncode=returncode, stdin=stdin) diff --git a/tests/integration/metrics_test.py b/tests/integration/metrics_test.py new file mode 100644 index 000000000..3d6e3fe22 --- /dev/null +++ b/tests/integration/metrics_test.py @@ -0,0 +1,125 @@ +import logging +import os +import socket +from http.server import BaseHTTPRequestHandler +from http.server import HTTPServer +from threading import Thread + +import requests +from docker.transport import UnixHTTPAdapter + +from tests.acceptance.cli_test import dispatch +from tests.integration.testcases import DockerClientTestCase + + +TEST_SOCKET_FILE = '/tmp/test-metrics-docker-cli.sock' + + +class MetricsTest(DockerClientTestCase): + test_session = requests.sessions.Session() + test_env = None + base_dir = 'tests/fixtures/v3-full' + + @classmethod + def setUpClass(cls): + super().setUpClass() + MetricsTest.test_session.mount("http+unix://", UnixHTTPAdapter(TEST_SOCKET_FILE)) + MetricsTest.test_env = os.environ.copy() + MetricsTest.test_env['METRICS_SOCKET_FILE'] = TEST_SOCKET_FILE + MetricsServer().start() + + @classmethod + def test_metrics_help(cls): + # root `docker-compose` command is considered as a `--help` + dispatch(cls.base_dir, [], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose --help", "context": "moby", ' \ + b'"source": "docker-compose", "status": "success"}' + dispatch(cls.base_dir, ['help', 'run'], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose help", "context": "moby", ' \ + b'"source": "docker-compose", "status": "success"}' + dispatch(cls.base_dir, ['--help'], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose --help", "context": "moby", ' \ + b'"source": "docker-compose", "status": "success"}' + dispatch(cls.base_dir, ['run', '--help'], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose --help run", "context": "moby", ' \ + b'"source": "docker-compose", "status": "success"}' + dispatch(cls.base_dir, ['up', '--help', 'extra_args'], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose --help up", "context": "moby", ' \ + b'"source": "docker-compose", "status": "success"}' + + @classmethod + def test_metrics_simple_commands(cls): + dispatch(cls.base_dir, ['ps'], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose ps", "context": "moby", ' \ + b'"source": "docker-compose", "status": "success"}' + dispatch(cls.base_dir, ['version'], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose version", "context": "moby", ' \ + b'"source": "docker-compose", "status": "success"}' + dispatch(cls.base_dir, ['version', '--yyy'], env=MetricsTest.test_env) + assert cls.get_content() == \ + b'{"command": "compose version", "context": "moby", ' \ + b'"source": "docker-compose", "status": "failure"}' + + @staticmethod + def get_content(): + resp = MetricsTest.test_session.get("http+unix://localhost") + print(resp.content) + return resp.content + + +def start_server(uri=TEST_SOCKET_FILE): + try: + os.remove(uri) + except OSError: + pass + httpd = HTTPServer(uri, MetricsHTTPRequestHandler, False) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(TEST_SOCKET_FILE) + sock.listen(0) + httpd.socket = sock + print('Serving on ', uri) + httpd.serve_forever() + sock.shutdown(socket.SHUT_RDWR) + sock.close() + os.remove(uri) + + +class MetricsServer: + @classmethod + def start(cls): + t = Thread(target=start_server, daemon=True) + t.start() + + +class MetricsHTTPRequestHandler(BaseHTTPRequestHandler): + usages = [] + + def do_GET(self): + self.client_address = ('',) # avoid exception in BaseHTTPServer.py log_message() + self.send_response(200) + self.end_headers() + for u in MetricsHTTPRequestHandler.usages: + self.wfile.write(u) + MetricsHTTPRequestHandler.usages = [] + + def do_POST(self): + self.client_address = ('',) # avoid exception in BaseHTTPServer.py log_message() + content_length = int(self.headers['Content-Length']) + body = self.rfile.read(content_length) + print(body) + MetricsHTTPRequestHandler.usages.append(body) + self.send_response(200) + self.end_headers() + + +if __name__ == '__main__': + logging.getLogger("urllib3").propagate = False + logging.getLogger("requests").propagate = False + start_server() diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 742d0e1c2..d4fbc9f61 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -61,6 +61,7 @@ class DockerClientTestCase(unittest.TestCase): @classmethod def tearDownClass(cls): + cls.client.close() del cls.client def tearDown(self): diff --git a/tests/unit/metrics/__init__.py b/tests/unit/metrics/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/metrics/metrics_test.py b/tests/unit/metrics/metrics_test.py new file mode 100644 index 000000000..e9f23720a --- /dev/null +++ b/tests/unit/metrics/metrics_test.py @@ -0,0 +1,36 @@ +import unittest + +from compose.metrics.client import MetricsCommand +from compose.metrics.client import Status + + +class MetricsTest(unittest.TestCase): + @classmethod + def test_metrics(cls): + assert MetricsCommand('up', 'moby').to_map() == { + 'command': 'compose up', + 'context': 'moby', + 'status': 'success', + 'source': 'docker-compose', + } + + assert MetricsCommand('down', 'local').to_map() == { + 'command': 'compose down', + 'context': 'local', + 'status': 'success', + 'source': 'docker-compose', + } + + assert MetricsCommand('help', 'aci', Status.FAILURE).to_map() == { + 'command': 'compose help', + 'context': 'aci', + 'status': 'failure', + 'source': 'docker-compose', + } + + assert MetricsCommand('run', 'ecs').to_map() == { + 'command': 'compose run', + 'context': 'ecs', + 'status': 'success', + 'source': 'docker-compose', + }