From 73c523a605d455eacee45d7cb811dfba45103e8b Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Sat, 16 Dec 2017 23:06:43 -0500 Subject: buildbot: add Python 3 support --- nixos/doc/manual/release-notes/rl-1903.xml | 9 + .../continuous-integration/buildbot/master.nix | 67 +++++-- .../continuous-integration/buildbot/worker.nix | 77 ++++++-- nixos/release.nix | 2 +- nixos/tests/buildbot.nix | 216 +++++++++++---------- 5 files changed, 236 insertions(+), 135 deletions(-) (limited to 'nixos') diff --git a/nixos/doc/manual/release-notes/rl-1903.xml b/nixos/doc/manual/release-notes/rl-1903.xml index 9cb5b93f27cb..c4847b60e274 100644 --- a/nixos/doc/manual/release-notes/rl-1903.xml +++ b/nixos/doc/manual/release-notes/rl-1903.xml @@ -113,6 +113,15 @@ (i.e. users.users.yourusername.extraGroups = ["video"];). + + + Buildbot now supports Python 3 and its packages have been moved to + pythonPackages. The options + and + can be used to select + the Python 2 or 3 version of the package. + + diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix index 8d767de37f00..0f07e6133bb5 100644 --- a/nixos/modules/services/continuous-integration/buildbot/master.nix +++ b/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -6,8 +6,12 @@ with lib; let cfg = config.services.buildbot-master; + + python = cfg.package.pythonModule; + escapeStr = s: escape ["'"] s; - masterCfg = if cfg.masterCfg == null then pkgs.writeText "master.cfg" '' + + defaultMasterCfg = pkgs.writeText "master.cfg" '' from buildbot.plugins import * factory = util.BuildFactory() c = BuildmasterConfig = dict( @@ -27,8 +31,28 @@ let factory.addStep(step) ${cfg.extraConfig} - '' - else cfg.masterCfg; + ''; + + tacFile = pkgs.writeText "buildbot-master.tac" '' + import os + + from twisted.application import service + from buildbot.master import BuildMaster + + basedir = '${cfg.buildbotDir}' + + configfile = '${cfg.masterCfg}' + + # Default umask for server + umask = None + + # note: this line is matched against to check that this is a buildmaster + # directory; do not edit it. + application = service.Application('buildmaster') + + m = BuildMaster(basedir, configfile, umask) + m.setServiceParent(application) + ''; in { options = { @@ -66,9 +90,9 @@ in { }; masterCfg = mkOption { - type = types.nullOr types.path; + type = types.path; description = "Optionally pass master.cfg path. Other options in this configuration will be ignored."; - default = null; + default = defaultMasterCfg; example = "/etc/nixos/buildbot/master.cfg"; }; @@ -175,18 +199,25 @@ in { package = mkOption { type = types.package; - default = pkgs.buildbot-full; - defaultText = "pkgs.buildbot-full"; + default = pkgs.pythonPackages.buildbot-full; + defaultText = "pkgs.pythonPackages.buildbot-full"; description = "Package to use for buildbot."; - example = literalExample "pkgs.buildbot-full"; + example = literalExample "pkgs.python3Packages.buildbot-full"; }; packages = mkOption { - default = with pkgs; [ python27Packages.twisted git ]; + default = [ pkgs.git ]; example = literalExample "[ pkgs.git ]"; type = types.listOf types.package; description = "Packages to add to PATH for the buildbot process."; }; + + pythonPackages = mkOption { + default = pythonPackages: with pythonPackages; [ ]; + defaultText = "pythonPackages: with pythonPackages; [ ]"; + description = "Packages to add the to the PYTHONPATH of the buildbot process."; + example = literalExample "pythonPackages: with pythonPackages; [ requests ]"; + }; }; }; @@ -210,14 +241,15 @@ in { description = "Buildbot Continuous Integration Server."; after = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; - path = cfg.packages; + path = cfg.packages ++ cfg.pythonPackages python.pkgs; + environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}"; preStart = '' - env > envvars - mkdir -vp ${cfg.buildbotDir} - ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg - rm -fv $cfg.buildbotDir}/buildbot.tac - ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir} + mkdir -vp "${cfg.buildbotDir}" + # Link the tac file so buildbot command line tools recognize the directory + ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac" + ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}" + rm -f buildbot.tac.new master.cfg.sample ''; serviceConfig = { @@ -225,12 +257,11 @@ in { User = cfg.user; Group = cfg.group; WorkingDirectory = cfg.home; - ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}"; + # NOTE: call twistd directly with stdout logging for systemd + ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${tacFile}"; }; - }; }; meta.maintainers = with lib.maintainers; [ nand0p mic92 ]; - } diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix index 67c541570b97..4130ec918a70 100644 --- a/nixos/modules/services/continuous-integration/buildbot/worker.nix +++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -7,6 +7,40 @@ with lib; let cfg = config.services.buildbot-worker; + python = cfg.package.pythonModule; + + tacFile = pkgs.writeText "aur-buildbot-worker.tac" '' + import os + from io import open + + from buildbot_worker.bot import Worker + from twisted.application import service + + basedir = '${cfg.buildbotDir}' + + # note: this line is matched against to check that this is a worker + # directory; do not edit it. + application = service.Application('buildbot-worker') + + master_url_split = '${cfg.masterUrl}'.split(':') + buildmaster_host = master_url_split[0] + port = int(master_url_split[1]) + workername = '${cfg.workerUser}' + + with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file: + passwd = passwd_file.read().strip('\r\n') + keepalive = 600 + umask = None + maxdelay = 300 + numcpus = None + allow_shutdown = None + + s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + numcpus=numcpus, allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + in { options = { services.buildbot-worker = { @@ -59,6 +93,23 @@ in { description = "Specifies the Buildbot Worker password."; }; + workerPassFile = mkOption { + type = types.path; + description = "File used to store the Buildbot Worker password"; + }; + + hostMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Description of this worker"; + }; + + adminMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Name of the administrator of this worker"; + }; + masterUrl = mkOption { default = "localhost:9989"; type = types.str; @@ -67,23 +118,24 @@ in { package = mkOption { type = types.package; - default = pkgs.buildbot-worker; - defaultText = "pkgs.buildbot-worker"; + default = pkgs.pythonPackages.buildbot-worker; + defaultText = "pkgs.pythonPackages.buildbot-worker"; description = "Package to use for buildbot worker."; - example = literalExample "pkgs.buildbot-worker"; + example = literalExample "pkgs.python3Packages.buildbot-worker"; }; packages = mkOption { - default = with pkgs; [ python27Packages.twisted git ]; + default = with pkgs; [ git ]; example = literalExample "[ pkgs.git ]"; type = types.listOf types.package; description = "Packages to add to PATH for the buildbot process."; }; - }; }; config = mkIf cfg.enable { + services.buildbot-worker.workerPassFile = mkDefault (pkgs.writeText "buildbot-worker-password" cfg.workerPass); + users.groups = optional (cfg.group == "bbworker") { name = "bbworker"; }; @@ -104,11 +156,16 @@ in { after = [ "network.target" "buildbot-master.service" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages; + environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}"; preStart = '' - mkdir -vp ${cfg.buildbotDir} - rm -fv $cfg.buildbotDir}/buildbot.tac - ${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass} + mkdir -vp "${cfg.buildbotDir}/info" + ${optionalString (cfg.hostMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host" + ''} + ${optionalString (cfg.adminMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin" + ''} ''; serviceConfig = { @@ -116,11 +173,9 @@ in { User = cfg.user; Group = cfg.group; WorkingDirectory = cfg.home; - Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages"; # NOTE: call twistd directly with stdout logging for systemd - #ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}"; - ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac"; + ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}"; }; }; diff --git a/nixos/release.nix b/nixos/release.nix index 86383f502a72..5412080cca18 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -257,7 +257,7 @@ in rec { tests.boot = callSubTests tests/boot.nix {}; tests.boot-stage1 = callTest tests/boot-stage1.nix {}; tests.borgbackup = callTest tests/borgbackup.nix {}; - tests.buildbot = callTest tests/buildbot.nix {}; + tests.buildbot = callSubTests tests/buildbot.nix {}; tests.cadvisor = callTestOnMatchingSystems ["x86_64-linux"] tests/cadvisor.nix {}; tests.ceph = callTestOnMatchingSystems ["x86_64-linux"] tests/ceph.nix {}; tests.certmgr = callSubTests tests/certmgr.nix {}; diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix index cf408dc7fec9..399fd39005e2 100644 --- a/nixos/tests/buildbot.nix +++ b/nixos/tests/buildbot.nix @@ -1,111 +1,117 @@ -# Test ensures buildbot master comes up correctly and workers can connect - -import ./make-test.nix ({ pkgs, ... } : { - name = "buildbot"; - - nodes = { - bbmaster = { pkgs, ... }: { - services.buildbot-master = { - enable = true; - package = pkgs.buildbot-full; - - # NOTE: use fake repo due to no internet in hydra ci - factorySteps = [ - "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')" - "steps.ShellCommand(command=['bash', 'fakerepo.sh'])" - ]; - changeSource = [ - "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" - ]; +{ system ? builtins.currentSystem }: + +with import ../lib/testing.nix { inherit system; }; + +let + # Test ensures buildbot master comes up correctly and workers can connect + mkBuildbotTest = python: makeTest { + name = "buildbot"; + + nodes = { + bbmaster = { pkgs, ... }: { + services.buildbot-master = { + enable = true; + package = python.pkgs.buildbot-full; + + # NOTE: use fake repo due to no internet in hydra ci + factorySteps = [ + "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')" + "steps.ShellCommand(command=['bash', 'fakerepo.sh'])" + ]; + changeSource = [ + "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" + ]; + }; + networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ]; + environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-full ]; }; - networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ]; - environment.systemPackages = with pkgs; [ git buildbot-full ]; - }; - bbworker = { pkgs, ... }: { - services.buildbot-worker = { - enable = true; - masterUrl = "bbmaster:9989"; + bbworker = { pkgs, ... }: { + services.buildbot-worker = { + enable = true; + masterUrl = "bbmaster:9989"; + }; + environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-worker ]; }; - environment.systemPackages = with pkgs; [ git buildbot-worker ]; - }; - gitrepo = { pkgs, ... }: { - services.openssh.enable = true; - networking.firewall.allowedTCPPorts = [ 22 9418 ]; - environment.systemPackages = with pkgs; [ git ]; + gitrepo = { pkgs, ... }: { + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 22 9418 ]; + environment.systemPackages = with pkgs; [ git ]; + }; }; - }; - testScript = '' - #Start up and populate fake repo - $gitrepo->waitForUnit("multi-user.target"); - print($gitrepo->execute(" \ - git config --global user.name 'Nobody Fakeuser' && \ - git config --global user.email 'nobody\@fakerepo.com' && \ - rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \ - mkdir -pv /srv/repos/fakerepo ~/.ssh && \ - ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \ - cat ~/.ssh/known_hosts && \ - cd /srv/repos/fakerepo && \ - git init && \ - echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \ - cat fakerepo.sh && \ - touch .git/git-daemon-export-ok && \ - git add fakerepo.sh .git/git-daemon-export-ok && \ - git commit -m fakerepo && \ - git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \ - ")); - - # Test gitrepo - $bbmaster->waitForUnit("network-online.target"); - #$bbmaster->execute("nc -z gitrepo 9418"); - print($bbmaster->execute(" \ - rm -rfv /tmp/fakerepo && \ - git clone git://gitrepo/fakerepo /tmp/fakerepo && \ - pwd && \ - ls -la && \ - ls -la /tmp/fakerepo \ - ")); - - # Test start master and connect worker - $bbmaster->waitForUnit("buildbot-master.service"); - $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/; - $bbworker->waitForUnit("network-online.target"); - $bbworker->execute("nc -z bbmaster 8010"); - $bbworker->execute("nc -z bbmaster 9989"); - $bbworker->waitForUnit("buildbot-worker.service"); - print($bbworker->execute("ls -la /home/bbworker/worker")); - - - # Test stop buildbot master and worker - print($bbmaster->execute(" \ - systemctl -l --no-pager status buildbot-master && \ - systemctl stop buildbot-master \ - ")); - $bbworker->fail("nc -z bbmaster 8010"); - $bbworker->fail("nc -z bbmaster 9989"); - print($bbworker->execute(" \ - systemctl -l --no-pager status buildbot-worker && \ - systemctl stop buildbot-worker && \ - ls -la /home/bbworker/worker \ - ")); - - - # Test buildbot daemon mode - # NOTE: daemon mode tests disabled due to broken PYTHONPATH child inheritence - # - #$bbmaster->execute("buildbot create-master /tmp"); - #$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg"); - #$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg"); - #$bbmaster->execute("buildbot start /tmp"); - #$bbworker->execute("nc -z bbmaster 8011"); - #$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/; - #$bbmaster->execute("buildbot stop /tmp"); - #$bbworker->fail("nc -z bbmaster 8011"); - - ''; - - meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ]; - -}) + testScript = '' + #Start up and populate fake repo + $gitrepo->waitForUnit("multi-user.target"); + print($gitrepo->execute(" \ + git config --global user.name 'Nobody Fakeuser' && \ + git config --global user.email 'nobody\@fakerepo.com' && \ + rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \ + mkdir -pv /srv/repos/fakerepo ~/.ssh && \ + ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \ + cat ~/.ssh/known_hosts && \ + cd /srv/repos/fakerepo && \ + git init && \ + echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \ + cat fakerepo.sh && \ + touch .git/git-daemon-export-ok && \ + git add fakerepo.sh .git/git-daemon-export-ok && \ + git commit -m fakerepo && \ + git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \ + ")); + + # Test gitrepo + $bbmaster->waitForUnit("network-online.target"); + #$bbmaster->execute("nc -z gitrepo 9418"); + print($bbmaster->execute(" \ + rm -rfv /tmp/fakerepo && \ + git clone git://gitrepo/fakerepo /tmp/fakerepo && \ + pwd && \ + ls -la && \ + ls -la /tmp/fakerepo \ + ")); + + # Test start master and connect worker + $bbmaster->waitForUnit("buildbot-master.service"); + $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/; + $bbworker->waitForUnit("network-online.target"); + $bbworker->execute("nc -z bbmaster 8010"); + $bbworker->execute("nc -z bbmaster 9989"); + $bbworker->waitForUnit("buildbot-worker.service"); + print($bbworker->execute("ls -la /home/bbworker/worker")); + + + # Test stop buildbot master and worker + print($bbmaster->execute(" \ + systemctl -l --no-pager status buildbot-master && \ + systemctl stop buildbot-master \ + ")); + $bbworker->fail("nc -z bbmaster 8010"); + $bbworker->fail("nc -z bbmaster 9989"); + print($bbworker->execute(" \ + systemctl -l --no-pager status buildbot-worker && \ + systemctl stop buildbot-worker && \ + ls -la /home/bbworker/worker \ + ")); + + + # Test buildbot daemon mode + $bbmaster->execute("buildbot create-master /tmp"); + $bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg"); + $bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg"); + $bbmaster->execute("buildbot start /tmp"); + $bbworker->execute("nc -z bbmaster 8011"); + $bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/; + $bbmaster->execute("buildbot stop /tmp"); + $bbworker->fail("nc -z bbmaster 8011"); + + ''; + + meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ]; + + }; +in { + python2 = mkBuildbotTest pkgs.python2; + python3 = mkBuildbotTest pkgs.python3; +} -- cgit 1.4.1