From 7ebcada02028e5ce8199cc123fda6aa1aba72e64 Mon Sep 17 00:00:00 2001 From: Charles Strahan Date: Wed, 28 Dec 2016 00:19:51 -0500 Subject: mesos: 1.0.1 -> 1.1.0 --- nixos/modules/services/misc/mesos-master.nix | 21 ++++++ nixos/modules/services/misc/mesos-slave.nix | 98 ++++++++++++++++++++++++++-- nixos/tests/mesos.nix | 97 +++++++++++++++++++++------ nixos/tests/mesos_test.py | 72 ++++++++++++++++++++ 4 files changed, 265 insertions(+), 23 deletions(-) create mode 100644 nixos/tests/mesos_test.py (limited to 'nixos') diff --git a/nixos/modules/services/misc/mesos-master.nix b/nixos/modules/services/misc/mesos-master.nix index 99583ebeebd5..0523c6549ed6 100644 --- a/nixos/modules/services/misc/mesos-master.nix +++ b/nixos/modules/services/misc/mesos-master.nix @@ -16,12 +16,30 @@ in { type = types.bool; }; + ip = mkOption { + description = "IP address to listen on."; + default = "0.0.0.0"; + type = types.str; + }; + port = mkOption { description = "Mesos Master port"; default = 5050; type = types.int; }; + advertiseIp = mkOption { + description = "IP address advertised to reach this master."; + default = null; + type = types.nullOr types.str; + }; + + advertisePort = mkOption { + description = "Port advertised to reach this Mesos master."; + default = null; + type = types.nullOr types.int; + }; + zk = mkOption { description = '' ZooKeeper URL (used for leader election amongst masters). @@ -84,7 +102,10 @@ in { serviceConfig = { ExecStart = '' ${pkgs.mesos}/bin/mesos-master \ + --ip=${cfg.ip} \ --port=${toString cfg.port} \ + ${optionalString (cfg.advertiseIp != null) "--advertise_ip=${cfg.advertiseIp}"} \ + ${optionalString (cfg.advertisePort != null) "--advertise_port=${toString cfg.advertisePort}"} \ ${if cfg.quorum == 0 then "--registry=in_memory" else "--zk=${cfg.zk} --registry=replicated_log --quorum=${toString cfg.quorum}"} \ diff --git a/nixos/modules/services/misc/mesos-slave.nix b/nixos/modules/services/misc/mesos-slave.nix index 9ddecb6fe30c..47be10274d3b 100644 --- a/nixos/modules/services/misc/mesos-slave.nix +++ b/nixos/modules/services/misc/mesos-slave.nix @@ -12,7 +12,23 @@ let attribsArg = optionalString (cfg.attributes != {}) "--attributes=${mkAttributes cfg.attributes}"; - containerizers = [ "mesos" ] ++ (optional cfg.withDocker "docker"); + containerizersArg = concatStringsSep "," ( + lib.unique ( + cfg.containerizers ++ (optional cfg.withDocker "docker") + ) + ); + + imageProvidersArg = concatStringsSep "," ( + lib.unique ( + cfg.imageProviders ++ (optional cfg.withDocker "docker") + ) + ); + + isolationArg = concatStringsSep "," ( + lib.unique ( + cfg.isolation ++ (optionals cfg.withDocker [ "filesystem/linux" "docker/runtime"]) + ) + ); in { @@ -27,7 +43,7 @@ in { ip = mkOption { description = "IP address to listen on."; default = "0.0.0.0"; - type = types.string; + type = types.str; }; port = mkOption { @@ -36,6 +52,53 @@ in { type = types.int; }; + advertiseIp = mkOption { + description = "IP address advertised to reach this agent."; + default = null; + type = types.nullOr types.str; + }; + + advertisePort = mkOption { + description = "Port advertised to reach this agent."; + default = null; + type = types.nullOr types.int; + }; + + containerizers = mkOption { + description = '' + List of containerizer implementations to compose in order to provide + containerization. Available options are mesos and docker. + The order the containerizers are specified is the order they are tried. + ''; + default = [ "mesos" ]; + type = types.listOf types.str; + }; + + imageProviders = mkOption { + description = "List of supported image providers, e.g., APPC,DOCKER."; + default = [ ]; + type = types.listOf types.str; + }; + + imageProvisionerBackend = mkOption { + description = '' + Strategy for provisioning container rootfs from images, + e.g., aufs, bind, copy, overlay. + ''; + default = "copy"; + type = types.str; + }; + + isolation = mkOption { + description = '' + Isolation mechanisms to use, e.g., posix/cpu,posix/mem, or + cgroups/cpu,cgroups/mem, or network/port_mapping, or `gpu/nvidia` for nvidia + specific gpu isolation. + ''; + default = [ "posix/cpu" "posix/mem" ]; + type = types.listOf types.str; + }; + master = mkOption { description = '' May be one of: @@ -57,6 +120,16 @@ in { type = types.bool; }; + dockerRegistry = mkOption { + description = '' + The default url for pulling Docker images. + It could either be a Docker registry server url, + or a local path in which Docker image archives are stored. + ''; + default = null; + type = types.nullOr (types.either types.str types.path); + }; + workDir = mkOption { description = "The Mesos work directory."; default = "/var/lib/mesos/slave"; @@ -96,28 +169,45 @@ in { host = "aabc123"; os = "nixos"; }; }; + + executorEnvironmentVariables = mkOption { + description = '' + The environment variables that should be passed to the executor, and thus subsequently task(s). + ''; + default = { + PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + }; + type = types.attrsOf types.str; + }; }; }; - config = mkIf cfg.enable { systemd.services.mesos-slave = { description = "Mesos Slave"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; - environment.MESOS_CONTAINERIZERS = concatStringsSep "," containerizers; + path = [ pkgs.stdenv.shellPackage ]; serviceConfig = { ExecStart = '' ${pkgs.mesos}/bin/mesos-slave \ + --containerizers=${containerizersArg} \ + --image_providers=${imageProvidersArg} \ + --image_provisioner_backend=${cfg.imageProvisionerBackend} \ + --isolation=${isolationArg} \ --ip=${cfg.ip} \ --port=${toString cfg.port} \ + ${optionalString (cfg.advertiseIp != null) "--advertise_ip=${cfg.advertiseIp}"} \ + ${optionalString (cfg.advertisePort != null) "--advertise_port=${toString cfg.advertisePort}"} \ --master=${cfg.master} \ --work_dir=${cfg.workDir} \ --logging_level=${cfg.logLevel} \ ${attribsArg} \ ${optionalString cfg.withHadoop "--hadoop-home=${pkgs.hadoop}"} \ ${optionalString cfg.withDocker "--docker=${pkgs.docker}/libexec/docker/docker"} \ + ${optionalString (cfg.dockerRegistry != null) "--docker_registry=${cfg.dockerRegistry}"} \ + --executor_environment_variables=${lib.escapeShellArg (builtins.toJSON cfg.executorEnvironmentVariables)} \ ${toString cfg.extraCmdLineOptions} ''; PermissionsStartOnly = true; diff --git a/nixos/tests/mesos.nix b/nixos/tests/mesos.nix index 3610603aeba2..6e9af126f032 100644 --- a/nixos/tests/mesos.nix +++ b/nixos/tests/mesos.nix @@ -1,32 +1,91 @@ -import ./make-test.nix ({ pkgs, ...} : { - name = "simple"; +import ./make-test.nix ({ pkgs, ...} : rec { + name = "mesos"; meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ offline ]; + maintainers = [ offline kamilchm cstrahan ]; }; - machine = { config, pkgs, ... }: { - services.zookeeper.enable = true; - virtualisation.docker.enable = true; - services.mesos = { - slave = { - enable = true; - master = "zk://localhost:2181/mesos"; - attributes = { - tag1 = "foo"; - tag2 = "bar"; - }; + nodes = { + master = { config, pkgs, ... }: { + networking.firewall.enable = false; + services.zookeeper.enable = true; + services.mesos.master = { + enable = true; + zk = "zk://master:2181/mesos"; }; - master = { - enable = true; - zk = "zk://localhost:2181/mesos"; + }; + + slave = { config, pkgs, ... }: { + networking.firewall.enable = false; + networking.nat.enable = true; + virtualisation.docker.enable = true; + services.mesos = { + slave = { + enable = true; + master = "master:5050"; + dockerRegistry = registry; + executorEnvironmentVariables = { + PATH = "/run/current-system/sw/bin"; + }; + }; }; }; }; + simpleDocker = pkgs.dockerTools.buildImage { + name = "echo"; + contents = [ pkgs.stdenv.shellPackage pkgs.coreutils ]; + config = { + Env = [ + # When shell=true, mesos invokes "sh -c ''", so make sure "sh" is + # on the PATH. + "PATH=${pkgs.stdenv.shellPackage}/bin:${pkgs.coreutils}/bin" + ]; + Entrypoint = [ "echo" ]; + }; + }; + + registry = pkgs.runCommand "registry" { } '' + mkdir -p $out + cp ${simpleDocker} $out/echo:latest.tar + ''; + + testFramework = pkgs.pythonPackages.buildPythonPackage { + name = "mesos-tests"; + propagatedBuildInputs = [ pkgs.mesos ]; + catchConflicts = false; + src = ./mesos_test.py; + phases = [ "installPhase" "fixupPhase" ]; + installPhase = '' + mkdir $out + cp $src $out/mesos_test.py + chmod +x $out/mesos_test.py + + echo "done" > test.result + tar czf $out/test.tar.gz test.result + ''; + }; + testScript = '' startAll; - $machine->waitForUnit("mesos-master.service"); - $machine->waitForUnit("mesos-slave.service"); + $master->waitForUnit("mesos-master.service"); + $slave->waitForUnit("mesos-slave.service"); + + $master->waitForOpenPort(5050); + $slave->waitForOpenPort(5051); + + # is slave registred? + $master->waitUntilSucceeds("curl -s --fail http://master:5050/master/slaves". + " | grep -q \"\\\"hostname\\\":\\\"slave\\\"\""); + + # try to run docker image + $master->succeed("${pkgs.mesos}/bin/mesos-execute --master=master:5050". + " --resources=\"cpus:0.1;mem:32\" --name=simple-docker". + " --containerizer=mesos --docker_image=echo:latest". + " --shell=true --command=\"echo done\" | grep -q TASK_FINISHED"); + + # simple command with .tar.gz uri + $master->succeed("${testFramework}/mesos_test.py master ". + "${testFramework}/test.tar.gz"); ''; }) diff --git a/nixos/tests/mesos_test.py b/nixos/tests/mesos_test.py new file mode 100644 index 000000000000..be8bb32e49a7 --- /dev/null +++ b/nixos/tests/mesos_test.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +import uuid +import time +import subprocess +import os + +import sys + +from mesos.interface import Scheduler +from mesos.native import MesosSchedulerDriver +from mesos.interface import mesos_pb2 + +def log(msg): + process = subprocess.Popen("systemd-cat", stdin=subprocess.PIPE) + (out,err) = process.communicate(msg) + +class NixosTestScheduler(Scheduler): + def __init__(self): + self.master_ip = sys.argv[1] + self.download_uri = sys.argv[2] + + def resourceOffers(self, driver, offers): + log("XXX got resource offer") + + offer = offers[0] + task = self.new_task(offer) + uri = task.command.uris.add() + uri.value = self.download_uri + task.command.value = "cat test.result" + driver.launchTasks(offer.id, [task]) + + def statusUpdate(self, driver, update): + log("XXX status update") + if update.state == mesos_pb2.TASK_FAILED: + log("XXX test task failed with message: " + update.message) + driver.stop() + sys.exit(1) + elif update.state == mesos_pb2.TASK_FINISHED: + driver.stop() + sys.exit(0) + + def new_task(self, offer): + task = mesos_pb2.TaskInfo() + id = uuid.uuid4() + task.task_id.value = str(id) + task.slave_id.value = offer.slave_id.value + task.name = "task {}".format(str(id)) + + cpus = task.resources.add() + cpus.name = "cpus" + cpus.type = mesos_pb2.Value.SCALAR + cpus.scalar.value = 0.1 + + mem = task.resources.add() + mem.name = "mem" + mem.type = mesos_pb2.Value.SCALAR + mem.scalar.value = 32 + + return task + +if __name__ == '__main__': + log("XXX framework started") + + framework = mesos_pb2.FrameworkInfo() + framework.user = "root" + framework.name = "nixos-test-framework" + driver = MesosSchedulerDriver( + NixosTestScheduler(), + framework, + sys.argv[1] + ":5050" + ) + driver.run() -- cgit 1.4.1