From 73c523a605d455eacee45d7cb811dfba45103e8b Mon Sep 17 00:00:00 2001 From: Ben Wolsieffer Date: Sat, 16 Dec 2017 23:06:43 -0500 Subject: [PATCH] buildbot: add Python 3 support --- nixos/doc/manual/release-notes/rl-1903.xml | 9 + .../buildbot/master.nix | 67 ++++-- .../buildbot/worker.nix | 77 ++++++- nixos/release.nix | 2 +- nixos/tests/buildbot.nix | 210 +++++++++--------- .../buildbot/default.nix | 96 ++++---- .../buildbot/pkg.nix | 4 +- .../buildbot/plugins.nix | 55 ++--- .../buildbot/skip_test_linux_distro.patch | 0 .../buildbot/worker.nix | 13 +- pkgs/top-level/aliases.nix | 5 + pkgs/top-level/all-packages.nix | 15 -- pkgs/top-level/python-packages.nix | 7 + 13 files changed, 331 insertions(+), 229 deletions(-) rename pkgs/development/{tools/build-managers => python-modules}/buildbot/default.nix (52%) rename pkgs/development/{tools/build-managers => python-modules}/buildbot/pkg.nix (87%) rename pkgs/development/{tools/build-managers => python-modules}/buildbot/plugins.nix (61%) rename pkgs/development/{tools/build-managers => python-modules}/buildbot/skip_test_linux_distro.patch (100%) rename pkgs/development/{tools/build-managers => python-modules}/buildbot/worker.nix (63%) diff --git a/nixos/doc/manual/release-notes/rl-1903.xml b/nixos/doc/manual/release-notes/rl-1903.xml index 9cb5b93f27c..c4847b60e27 100644 --- a/nixos/doc/manual/release-notes/rl-1903.xml +++ b/nixos/doc/manual/release-notes/rl-1903.xml @@ -113,6 +113,15 @@ (i.e. users.users.yourusername.extraGroups = ["video"];). + + + Buildbot now supports Python 3 and its packages have been moved to + pythonPackages. The options + and + can be used to select + the Python 2 or 3 version of the package. + + diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix index 8d767de37f0..0f07e6133bb 100644 --- a/nixos/modules/services/continuous-integration/buildbot/master.nix +++ b/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -6,8 +6,12 @@ with lib; let cfg = config.services.buildbot-master; + + python = cfg.package.pythonModule; + escapeStr = s: escape ["'"] s; - masterCfg = if cfg.masterCfg == null then pkgs.writeText "master.cfg" '' + + defaultMasterCfg = pkgs.writeText "master.cfg" '' from buildbot.plugins import * factory = util.BuildFactory() c = BuildmasterConfig = dict( @@ -27,8 +31,28 @@ let factory.addStep(step) ${cfg.extraConfig} - '' - else cfg.masterCfg; + ''; + + tacFile = pkgs.writeText "buildbot-master.tac" '' + import os + + from twisted.application import service + from buildbot.master import BuildMaster + + basedir = '${cfg.buildbotDir}' + + configfile = '${cfg.masterCfg}' + + # Default umask for server + umask = None + + # note: this line is matched against to check that this is a buildmaster + # directory; do not edit it. + application = service.Application('buildmaster') + + m = BuildMaster(basedir, configfile, umask) + m.setServiceParent(application) + ''; in { options = { @@ -66,9 +90,9 @@ in { }; masterCfg = mkOption { - type = types.nullOr types.path; + type = types.path; description = "Optionally pass master.cfg path. Other options in this configuration will be ignored."; - default = null; + default = defaultMasterCfg; example = "/etc/nixos/buildbot/master.cfg"; }; @@ -175,18 +199,25 @@ in { package = mkOption { type = types.package; - default = pkgs.buildbot-full; - defaultText = "pkgs.buildbot-full"; + default = pkgs.pythonPackages.buildbot-full; + defaultText = "pkgs.pythonPackages.buildbot-full"; description = "Package to use for buildbot."; - example = literalExample "pkgs.buildbot-full"; + example = literalExample "pkgs.python3Packages.buildbot-full"; }; packages = mkOption { - default = with pkgs; [ python27Packages.twisted git ]; + default = [ pkgs.git ]; example = literalExample "[ pkgs.git ]"; type = types.listOf types.package; description = "Packages to add to PATH for the buildbot process."; }; + + pythonPackages = mkOption { + default = pythonPackages: with pythonPackages; [ ]; + defaultText = "pythonPackages: with pythonPackages; [ ]"; + description = "Packages to add the to the PYTHONPATH of the buildbot process."; + example = literalExample "pythonPackages: with pythonPackages; [ requests ]"; + }; }; }; @@ -210,14 +241,15 @@ in { description = "Buildbot Continuous Integration Server."; after = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; - path = cfg.packages; + path = cfg.packages ++ cfg.pythonPackages python.pkgs; + environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}"; preStart = '' - env > envvars - mkdir -vp ${cfg.buildbotDir} - ln -sfv ${masterCfg} ${cfg.buildbotDir}/master.cfg - rm -fv $cfg.buildbotDir}/buildbot.tac - ${cfg.package}/bin/buildbot create-master ${cfg.buildbotDir} + mkdir -vp "${cfg.buildbotDir}" + # Link the tac file so buildbot command line tools recognize the directory + ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac" + ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}" + rm -f buildbot.tac.new master.cfg.sample ''; serviceConfig = { @@ -225,12 +257,11 @@ in { User = cfg.user; Group = cfg.group; WorkingDirectory = cfg.home; - ExecStart = "${cfg.package}/bin/buildbot start --nodaemon ${cfg.buildbotDir}"; + # NOTE: call twistd directly with stdout logging for systemd + ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${tacFile}"; }; - }; }; meta.maintainers = with lib.maintainers; [ nand0p mic92 ]; - } diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix index 67c541570b9..4130ec918a7 100644 --- a/nixos/modules/services/continuous-integration/buildbot/worker.nix +++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -7,6 +7,40 @@ with lib; let cfg = config.services.buildbot-worker; + python = cfg.package.pythonModule; + + tacFile = pkgs.writeText "aur-buildbot-worker.tac" '' + import os + from io import open + + from buildbot_worker.bot import Worker + from twisted.application import service + + basedir = '${cfg.buildbotDir}' + + # note: this line is matched against to check that this is a worker + # directory; do not edit it. + application = service.Application('buildbot-worker') + + master_url_split = '${cfg.masterUrl}'.split(':') + buildmaster_host = master_url_split[0] + port = int(master_url_split[1]) + workername = '${cfg.workerUser}' + + with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file: + passwd = passwd_file.read().strip('\r\n') + keepalive = 600 + umask = None + maxdelay = 300 + numcpus = None + allow_shutdown = None + + s = Worker(buildmaster_host, port, workername, passwd, basedir, + keepalive, umask=umask, maxdelay=maxdelay, + numcpus=numcpus, allow_shutdown=allow_shutdown) + s.setServiceParent(application) + ''; + in { options = { services.buildbot-worker = { @@ -59,6 +93,23 @@ in { description = "Specifies the Buildbot Worker password."; }; + workerPassFile = mkOption { + type = types.path; + description = "File used to store the Buildbot Worker password"; + }; + + hostMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Description of this worker"; + }; + + adminMessage = mkOption { + default = null; + type = types.nullOr types.str; + description = "Name of the administrator of this worker"; + }; + masterUrl = mkOption { default = "localhost:9989"; type = types.str; @@ -67,23 +118,24 @@ in { package = mkOption { type = types.package; - default = pkgs.buildbot-worker; - defaultText = "pkgs.buildbot-worker"; + default = pkgs.pythonPackages.buildbot-worker; + defaultText = "pkgs.pythonPackages.buildbot-worker"; description = "Package to use for buildbot worker."; - example = literalExample "pkgs.buildbot-worker"; + example = literalExample "pkgs.python3Packages.buildbot-worker"; }; packages = mkOption { - default = with pkgs; [ python27Packages.twisted git ]; + default = with pkgs; [ git ]; example = literalExample "[ pkgs.git ]"; type = types.listOf types.package; description = "Packages to add to PATH for the buildbot process."; }; - }; }; config = mkIf cfg.enable { + services.buildbot-worker.workerPassFile = mkDefault (pkgs.writeText "buildbot-worker-password" cfg.workerPass); + users.groups = optional (cfg.group == "bbworker") { name = "bbworker"; }; @@ -104,11 +156,16 @@ in { after = [ "network.target" "buildbot-master.service" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages; + environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}"; preStart = '' - mkdir -vp ${cfg.buildbotDir} - rm -fv $cfg.buildbotDir}/buildbot.tac - ${cfg.package}/bin/buildbot-worker create-worker ${cfg.buildbotDir} ${cfg.masterUrl} ${cfg.workerUser} ${cfg.workerPass} + mkdir -vp "${cfg.buildbotDir}/info" + ${optionalString (cfg.hostMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host" + ''} + ${optionalString (cfg.adminMessage != null) '' + ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin" + ''} ''; serviceConfig = { @@ -116,11 +173,9 @@ in { User = cfg.user; Group = cfg.group; WorkingDirectory = cfg.home; - Environment = "PYTHONPATH=${cfg.package}/lib/python2.7/site-packages:${pkgs.python27Packages.future}/lib/python2.7/site-packages"; # NOTE: call twistd directly with stdout logging for systemd - #ExecStart = "${cfg.package}/bin/buildbot-worker start --nodaemon ${cfg.buildbotDir}"; - ExecStart = "${pkgs.python27Packages.twisted}/bin/twistd -n -l - -y ${cfg.buildbotDir}/buildbot.tac"; + ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}"; }; }; diff --git a/nixos/release.nix b/nixos/release.nix index 86383f502a7..5412080cca1 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -257,7 +257,7 @@ in rec { tests.boot = callSubTests tests/boot.nix {}; tests.boot-stage1 = callTest tests/boot-stage1.nix {}; tests.borgbackup = callTest tests/borgbackup.nix {}; - tests.buildbot = callTest tests/buildbot.nix {}; + tests.buildbot = callSubTests tests/buildbot.nix {}; tests.cadvisor = callTestOnMatchingSystems ["x86_64-linux"] tests/cadvisor.nix {}; tests.ceph = callTestOnMatchingSystems ["x86_64-linux"] tests/ceph.nix {}; tests.certmgr = callSubTests tests/certmgr.nix {}; diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix index cf408dc7fec..399fd39005e 100644 --- a/nixos/tests/buildbot.nix +++ b/nixos/tests/buildbot.nix @@ -1,111 +1,117 @@ -# Test ensures buildbot master comes up correctly and workers can connect +{ system ? builtins.currentSystem }: -import ./make-test.nix ({ pkgs, ... } : { - name = "buildbot"; +with import ../lib/testing.nix { inherit system; }; - nodes = { - bbmaster = { pkgs, ... }: { - services.buildbot-master = { - enable = true; - package = pkgs.buildbot-full; +let + # Test ensures buildbot master comes up correctly and workers can connect + mkBuildbotTest = python: makeTest { + name = "buildbot"; - # NOTE: use fake repo due to no internet in hydra ci - factorySteps = [ - "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')" - "steps.ShellCommand(command=['bash', 'fakerepo.sh'])" - ]; - changeSource = [ - "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" - ]; + nodes = { + bbmaster = { pkgs, ... }: { + services.buildbot-master = { + enable = true; + package = python.pkgs.buildbot-full; + + # NOTE: use fake repo due to no internet in hydra ci + factorySteps = [ + "steps.Git(repourl='git://gitrepo/fakerepo.git', mode='incremental')" + "steps.ShellCommand(command=['bash', 'fakerepo.sh'])" + ]; + changeSource = [ + "changes.GitPoller('git://gitrepo/fakerepo.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)" + ]; + }; + networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ]; + environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-full ]; }; - networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ]; - environment.systemPackages = with pkgs; [ git buildbot-full ]; - }; - bbworker = { pkgs, ... }: { - services.buildbot-worker = { - enable = true; - masterUrl = "bbmaster:9989"; + bbworker = { pkgs, ... }: { + services.buildbot-worker = { + enable = true; + masterUrl = "bbmaster:9989"; + }; + environment.systemPackages = with pkgs; [ git python.pkgs.buildbot-worker ]; + }; + + gitrepo = { pkgs, ... }: { + services.openssh.enable = true; + networking.firewall.allowedTCPPorts = [ 22 9418 ]; + environment.systemPackages = with pkgs; [ git ]; }; - environment.systemPackages = with pkgs; [ git buildbot-worker ]; }; - gitrepo = { pkgs, ... }: { - services.openssh.enable = true; - networking.firewall.allowedTCPPorts = [ 22 9418 ]; - environment.systemPackages = with pkgs; [ git ]; - }; + testScript = '' + #Start up and populate fake repo + $gitrepo->waitForUnit("multi-user.target"); + print($gitrepo->execute(" \ + git config --global user.name 'Nobody Fakeuser' && \ + git config --global user.email 'nobody\@fakerepo.com' && \ + rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \ + mkdir -pv /srv/repos/fakerepo ~/.ssh && \ + ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \ + cat ~/.ssh/known_hosts && \ + cd /srv/repos/fakerepo && \ + git init && \ + echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \ + cat fakerepo.sh && \ + touch .git/git-daemon-export-ok && \ + git add fakerepo.sh .git/git-daemon-export-ok && \ + git commit -m fakerepo && \ + git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \ + ")); + + # Test gitrepo + $bbmaster->waitForUnit("network-online.target"); + #$bbmaster->execute("nc -z gitrepo 9418"); + print($bbmaster->execute(" \ + rm -rfv /tmp/fakerepo && \ + git clone git://gitrepo/fakerepo /tmp/fakerepo && \ + pwd && \ + ls -la && \ + ls -la /tmp/fakerepo \ + ")); + + # Test start master and connect worker + $bbmaster->waitForUnit("buildbot-master.service"); + $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/; + $bbworker->waitForUnit("network-online.target"); + $bbworker->execute("nc -z bbmaster 8010"); + $bbworker->execute("nc -z bbmaster 9989"); + $bbworker->waitForUnit("buildbot-worker.service"); + print($bbworker->execute("ls -la /home/bbworker/worker")); + + + # Test stop buildbot master and worker + print($bbmaster->execute(" \ + systemctl -l --no-pager status buildbot-master && \ + systemctl stop buildbot-master \ + ")); + $bbworker->fail("nc -z bbmaster 8010"); + $bbworker->fail("nc -z bbmaster 9989"); + print($bbworker->execute(" \ + systemctl -l --no-pager status buildbot-worker && \ + systemctl stop buildbot-worker && \ + ls -la /home/bbworker/worker \ + ")); + + + # Test buildbot daemon mode + $bbmaster->execute("buildbot create-master /tmp"); + $bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg"); + $bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg"); + $bbmaster->execute("buildbot start /tmp"); + $bbworker->execute("nc -z bbmaster 8011"); + $bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/; + $bbmaster->execute("buildbot stop /tmp"); + $bbworker->fail("nc -z bbmaster 8011"); + + ''; + + meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ]; + }; - - testScript = '' - #Start up and populate fake repo - $gitrepo->waitForUnit("multi-user.target"); - print($gitrepo->execute(" \ - git config --global user.name 'Nobody Fakeuser' && \ - git config --global user.email 'nobody\@fakerepo.com' && \ - rm -rvf /srv/repos/fakerepo.git /tmp/fakerepo && \ - mkdir -pv /srv/repos/fakerepo ~/.ssh && \ - ssh-keyscan -H gitrepo > ~/.ssh/known_hosts && \ - cat ~/.ssh/known_hosts && \ - cd /srv/repos/fakerepo && \ - git init && \ - echo -e '#!/bin/sh\necho fakerepo' > fakerepo.sh && \ - cat fakerepo.sh && \ - touch .git/git-daemon-export-ok && \ - git add fakerepo.sh .git/git-daemon-export-ok && \ - git commit -m fakerepo && \ - git daemon --verbose --export-all --base-path=/srv/repos --reuseaddr & \ - ")); - - # Test gitrepo - $bbmaster->waitForUnit("network-online.target"); - #$bbmaster->execute("nc -z gitrepo 9418"); - print($bbmaster->execute(" \ - rm -rfv /tmp/fakerepo && \ - git clone git://gitrepo/fakerepo /tmp/fakerepo && \ - pwd && \ - ls -la && \ - ls -la /tmp/fakerepo \ - ")); - - # Test start master and connect worker - $bbmaster->waitForUnit("buildbot-master.service"); - $bbmaster->waitUntilSucceeds("curl -s --head http://bbmaster:8010") =~ /200 OK/; - $bbworker->waitForUnit("network-online.target"); - $bbworker->execute("nc -z bbmaster 8010"); - $bbworker->execute("nc -z bbmaster 9989"); - $bbworker->waitForUnit("buildbot-worker.service"); - print($bbworker->execute("ls -la /home/bbworker/worker")); - - - # Test stop buildbot master and worker - print($bbmaster->execute(" \ - systemctl -l --no-pager status buildbot-master && \ - systemctl stop buildbot-master \ - ")); - $bbworker->fail("nc -z bbmaster 8010"); - $bbworker->fail("nc -z bbmaster 9989"); - print($bbworker->execute(" \ - systemctl -l --no-pager status buildbot-worker && \ - systemctl stop buildbot-worker && \ - ls -la /home/bbworker/worker \ - ")); - - - # Test buildbot daemon mode - # NOTE: daemon mode tests disabled due to broken PYTHONPATH child inheritence - # - #$bbmaster->execute("buildbot create-master /tmp"); - #$bbmaster->execute("mv -fv /tmp/master.cfg.sample /tmp/master.cfg"); - #$bbmaster->execute("sed -i 's/8010/8011/' /tmp/master.cfg"); - #$bbmaster->execute("buildbot start /tmp"); - #$bbworker->execute("nc -z bbmaster 8011"); - #$bbworker->waitUntilSucceeds("curl -s --head http://bbmaster:8011") =~ /200 OK/; - #$bbmaster->execute("buildbot stop /tmp"); - #$bbworker->fail("nc -z bbmaster 8011"); - - ''; - - meta.maintainers = with pkgs.stdenv.lib.maintainers; [ nand0p ]; - -}) +in { + python2 = mkBuildbotTest pkgs.python2; + python3 = mkBuildbotTest pkgs.python3; +} diff --git a/pkgs/development/tools/build-managers/buildbot/default.nix b/pkgs/development/python-modules/buildbot/default.nix similarity index 52% rename from pkgs/development/tools/build-managers/buildbot/default.nix rename to pkgs/development/python-modules/buildbot/default.nix index 7e89fdc348e..0110aa92c06 100644 --- a/pkgs/development/tools/build-managers/buildbot/default.nix +++ b/pkgs/development/python-modules/buildbot/default.nix @@ -1,72 +1,68 @@ -{ stdenv, openssh, buildbot-worker, buildbot-pkg, pythonPackages, runCommand, makeWrapper }: +{ stdenv, lib, buildPythonPackage, fetchPypi, makeWrapper, isPy3k, + python, twisted, jinja2, zope_interface, future, sqlalchemy, + sqlalchemy_migrate, dateutil, txaio, autobahn, pyjwt, treq, txrequests, + txgithub, pyjade, boto3, moto, mock, lz4, setuptoolsTrial, isort, pylint, + flake8, buildbot-worker, buildbot-pkg, glibcLocales }: let - withPlugins = plugins: runCommand "wrapped-${package.name}" { - buildInputs = [ makeWrapper ] ++ plugins; - propagatedBuildInputs = package.propagatedBuildInputs; - passthru.withPlugins = moarPlugins: withPlugins (moarPlugins ++ plugins); - } '' - makeWrapper ${package}/bin/buildbot $out/bin/buildbot \ - --prefix PYTHONPATH : "${package}/lib/python2.7/site-packages:$PYTHONPATH" - ln -sfv ${package}/lib $out/lib - ''; + withPlugins = plugins: buildPythonPackage { + name = "${package.name}-with-plugins"; + phases = [ "installPhase" "fixupPhase" ]; + buildInputs = [ makeWrapper ]; + propagatedBuildInputs = plugins ++ package.propagatedBuildInputs; - package = pythonPackages.buildPythonApplication rec { + installPhase = '' + makeWrapper ${package}/bin/buildbot $out/bin/buildbot \ + --prefix PYTHONPATH : "${package}/${python.sitePackages}:$PYTHONPATH" + ln -sfv ${package}/lib $out/lib + ''; + + passthru = package.passthru // { + withPlugins = morePlugins: withPlugins (morePlugins ++ plugins); + }; + }; + + package = buildPythonPackage rec { pname = "buildbot"; version = "1.4.0"; - src = pythonPackages.fetchPypi { + src = fetchPypi { inherit pname version; sha256 = "0dfa926nh642i3bnpzlz0q347zicyx6wswjfqbniri59ya64fncx"; }; - buildInputs = with pythonPackages; [ - lz4 - txrequests - pyjade - boto3 - moto - txgithub - mock - setuptoolsTrial - isort - pylint - astroid - pyflakes - openssh - buildbot-worker - buildbot-pkg - treq - ]; - - propagatedBuildInputs = with pythonPackages; [ + propagatedBuildInputs = [ # core twisted jinja2 zope_interface + future sqlalchemy sqlalchemy_migrate - future dateutil txaio autobahn pyjwt - distro # tls - pyopenssl - service-identity - idna - - # docs - sphinx - sphinxcontrib-blockdiag - sphinxcontrib-spelling - pyenchant - docutils - ramlfications - sphinx-jinja + twisted.extras.tls + ]; + checkInputs = [ + treq + txrequests + pyjade + boto3 + moto + mock + lz4 + setuptoolsTrial + isort + pylint + flake8 + buildbot-worker + buildbot-pkg + glibcLocales ]; patches = [ @@ -75,6 +71,8 @@ let ./skip_test_linux_distro.patch ]; + LC_ALL = "en_US.UTF-8"; + # TimeoutErrors on slow machines -> aarch64 doCheck = !stdenv.isAarch64; @@ -82,9 +80,11 @@ let substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" ''; - passthru = { inherit withPlugins; }; + passthru = { + inherit withPlugins; + }; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot is an open-source continuous integration framework for automating software build, test, and release processes"; maintainers = with maintainers; [ nand0p ryansydnor ]; diff --git a/pkgs/development/tools/build-managers/buildbot/pkg.nix b/pkgs/development/python-modules/buildbot/pkg.nix similarity index 87% rename from pkgs/development/tools/build-managers/buildbot/pkg.nix rename to pkgs/development/python-modules/buildbot/pkg.nix index 452c2ffea04..b9358b1c420 100644 --- a/pkgs/development/tools/build-managers/buildbot/pkg.nix +++ b/pkgs/development/python-modules/buildbot/pkg.nix @@ -1,4 +1,4 @@ -{ stdenv, buildPythonPackage, fetchPypi, setuptools }: +{ lib, buildPythonPackage, fetchPypi, setuptools }: buildPythonPackage rec { pname = "buildbot-pkg"; @@ -15,7 +15,7 @@ buildPythonPackage rec { substituteInPlace buildbot_pkg.py --replace "os.listdir = listdir" "" ''; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot Packaging Helper"; maintainers = with maintainers; [ nand0p ryansydnor ]; diff --git a/pkgs/development/tools/build-managers/buildbot/plugins.nix b/pkgs/development/python-modules/buildbot/plugins.nix similarity index 61% rename from pkgs/development/tools/build-managers/buildbot/plugins.nix rename to pkgs/development/python-modules/buildbot/plugins.nix index 91499b6e755..bdc67d178d4 100644 --- a/pkgs/development/tools/build-managers/buildbot/plugins.nix +++ b/pkgs/development/python-modules/buildbot/plugins.nix @@ -1,19 +1,19 @@ -{ stdenv, pythonPackages, buildbot-pkg }: +{ lib, buildPythonPackage, fetchPypi, buildbot, buildbot-pkg }: { - www = pythonPackages.buildPythonPackage rec { + www = buildPythonPackage rec { pname = "buildbot_www"; - version = buildbot-pkg.version; + inherit (buildbot-pkg) version; # NOTE: wheel is used due to buildbot circular dependency format = "wheel"; - src = pythonPackages.fetchPypi { + src = fetchPypi { inherit pname version format; sha256 = "1m5dsp1gn9m5vfh5hnqp8g6hmhw1f1ydnassd33nhk521f2akz0v"; }; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot UI"; maintainers = with maintainers; [ nand0p ryansydnor ]; @@ -21,18 +21,19 @@ }; }; - console-view = pythonPackages.buildPythonPackage rec { + console-view = buildPythonPackage rec { pname = "buildbot-console-view"; - version = buildbot-pkg.version; + inherit (buildbot-pkg) version; - src = pythonPackages.fetchPypi { + src = fetchPypi { inherit pname version; sha256 = "0vblaxmihgb4w9aa5q0wcgvxs7qzajql8s22w0pl9qs494g05s9r"; }; - propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; + propagatedBuildInputs = [ buildbot-pkg ]; + checkInputs = [ buildbot ]; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot Console View Plugin"; maintainers = with maintainers; [ nand0p ryansydnor ]; @@ -40,18 +41,19 @@ }; }; - waterfall-view = pythonPackages.buildPythonPackage rec { + waterfall-view = buildPythonPackage rec { pname = "buildbot-waterfall-view"; - version = buildbot-pkg.version; + inherit (buildbot-pkg) version; - src = pythonPackages.fetchPypi { + src = fetchPypi { inherit pname version; sha256 = "18v1a6dapwjc2s9hi0cv3ry3s048w84md908zwaa3033gz3zwzy7"; }; - propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; + propagatedBuildInputs = [ buildbot-pkg ]; + checkInputs = [ buildbot ]; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot Waterfall View Plugin"; maintainers = with maintainers; [ nand0p ryansydnor ]; @@ -59,18 +61,19 @@ }; }; - grid-view = pythonPackages.buildPythonPackage rec { + grid-view = buildPythonPackage rec { pname = "buildbot-grid-view"; - version = buildbot-pkg.version; + inherit (buildbot-pkg) version; - src = pythonPackages.fetchPypi { + src = fetchPypi { inherit pname version; sha256 = "0iawsy892v6rn88hsgiiwaf689jqzhnb2wbxh6zkz3c0hvq4g0qd"; }; - propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; + propagatedBuildInputs = [ buildbot-pkg ]; + checkInputs = [ buildbot ]; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot Grid View Plugin"; maintainers = with maintainers; [ nand0p ]; @@ -78,23 +81,23 @@ }; }; - wsgi-dashboards = pythonPackages.buildPythonPackage rec { + wsgi-dashboards = buildPythonPackage rec { pname = "buildbot-wsgi-dashboards"; - version = buildbot-pkg.version; + inherit (buildbot-pkg) version; - src = pythonPackages.fetchPypi { + src = fetchPypi { inherit pname version; sha256 = "00cpjna3bffh1qbq6a3sqffd1g7qhbrmn9gpzxf9k38jam6jgfpz"; }; - propagatedBuildInputs = with pythonPackages; [ buildbot-pkg ]; + propagatedBuildInputs = [ buildbot-pkg ]; + checkInputs = [ buildbot ]; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot WSGI dashboards Plugin"; maintainers = with maintainers; [ ]; license = licenses.gpl2; }; }; - } diff --git a/pkgs/development/tools/build-managers/buildbot/skip_test_linux_distro.patch b/pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch similarity index 100% rename from pkgs/development/tools/build-managers/buildbot/skip_test_linux_distro.patch rename to pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch diff --git a/pkgs/development/tools/build-managers/buildbot/worker.nix b/pkgs/development/python-modules/buildbot/worker.nix similarity index 63% rename from pkgs/development/tools/build-managers/buildbot/worker.nix rename to pkgs/development/python-modules/buildbot/worker.nix index 02beb686399..4e54276f8ae 100644 --- a/pkgs/development/tools/build-managers/buildbot/worker.nix +++ b/pkgs/development/python-modules/buildbot/worker.nix @@ -1,22 +1,23 @@ -{ stdenv, pythonPackages }: +{ lib, buildPythonPackage, fetchPypi, python, setuptoolsTrial, mock, twisted, future }: -pythonPackages.buildPythonApplication (rec { +buildPythonPackage (rec { pname = "buildbot-worker"; version = "1.4.0"; - src = pythonPackages.fetchPypi { + src = fetchPypi { inherit pname version; sha256 = "12zvf4c39b6s4g1f2w407q8kkw602m88rc1ggi4w9pkw3bwbxrgy"; }; - buildInputs = with pythonPackages; [ setuptoolsTrial mock ]; - propagatedBuildInputs = with pythonPackages; [ twisted future ]; + propagatedBuildInputs = [ twisted future ]; + + checkInputs = [ setuptoolsTrial mock ]; postPatch = '' substituteInPlace buildbot_worker/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" ''; - meta = with stdenv.lib; { + meta = with lib; { homepage = http://buildbot.net/; description = "Buildbot Worker Daemon"; maintainers = with maintainers; [ nand0p ryansydnor ]; diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index fe0d3816413..0abf0e60846 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -49,6 +49,11 @@ mapAliases ({ bashCompletion = bash-completion; # Added 2016-09-28 bridge_utils = bridge-utils; # added 2015-02-20 btrfsProgs = btrfs-progs; # added 2016-01-03 + buildbot = pythonPackages.buildbot; # added 2018-10-11 + buildbot-full = pythonPackages.buildbot-full; # added 2018-10-11 + buildbot-pkg = pythonPackages.buildbot-pkg; # added 2018-10-11 + buildbot-ui = pythonPackages.buildbot-ui; # added 2018-10-11 + buildbot-worker = pythonPackages.buildbot-worker; # added 2018-10-11 bundler_HEAD = bundler; # added 2015-11-15 cantarell_fonts = cantarell-fonts; # added 2018-03-03 checkbashism = checkbashisms; # added 2016-08-16 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index d911181ae37..b34501a8f69 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -8135,21 +8135,6 @@ with pkgs; buck = callPackage ../development/tools/build-managers/buck { }; - buildbot = callPackage ../development/tools/build-managers/buildbot { - pythonPackages = python2Packages; - }; - buildbot-worker = callPackage ../development/tools/build-managers/buildbot/worker.nix { - pythonPackages = python2Packages; - }; - buildbot-pkg = callPackage ../development/tools/build-managers/buildbot/pkg.nix { - inherit (python2Packages) buildPythonPackage fetchPypi setuptools; - }; - buildbot-plugins = callPackages ../development/tools/build-managers/buildbot/plugins.nix { - pythonPackages = python2Packages; - }; - buildbot-ui = buildbot.withPlugins (with self.buildbot-plugins; [ www ]); - buildbot-full = buildbot.withPlugins (with self.buildbot-plugins; [ www console-view waterfall-view grid-view wsgi-dashboards ]); - buildkite-agent = buildkite-agent2; buildkite-agent2 = callPackage ../development/tools/continuous-integration/buildkite-agent/2.x.nix { }; buildkite-agent3 = callPackage ../development/tools/continuous-integration/buildkite-agent/3.x.nix { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 73d2cf6d2ac..3822f08bad7 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1421,6 +1421,13 @@ in { bugzilla = callPackage ../development/python-modules/bugzilla { }; + buildbot = callPackage ../development/python-modules/buildbot { }; + buildbot-plugins = pkgs.recurseIntoAttrs (callPackage ../development/python-modules/buildbot/plugins.nix { }); + buildbot-ui = self.buildbot.withPlugins (with self.buildbot-plugins; [ www ]); + buildbot-full = self.buildbot.withPlugins (with self.buildbot-plugins; [ www console-view waterfall-view grid-view wsgi-dashboards ]); + buildbot-worker = callPackage ../development/python-modules/buildbot/worker.nix { }; + buildbot-pkg = callPackage ../development/python-modules/buildbot/pkg.nix { }; + check-manifest = callPackage ../development/python-modules/check-manifest { }; devpi-common = callPackage ../development/python-modules/devpi-common { };