From 4b9a2e13a82ea293e0d1530e8d32ca07074996dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 18 Aug 2020 09:33:18 +0100 Subject: [PATCH 1/5] cloud-utils: split of smaller .guest output --- nixos/modules/system/boot/grow-partition.nix | 2 +- pkgs/tools/misc/cloud-utils/default.nix | 30 ++++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/nixos/modules/system/boot/grow-partition.nix b/nixos/modules/system/boot/grow-partition.nix index 71a86c74772..be70c4ad9c8 100644 --- a/nixos/modules/system/boot/grow-partition.nix +++ b/nixos/modules/system/boot/grow-partition.nix @@ -23,7 +23,7 @@ with lib; copy_bin_and_libs ${pkgs.utillinux}/sbin/sfdisk copy_bin_and_libs ${pkgs.utillinux}/sbin/lsblk - substitute "${pkgs.cloud-utils}/bin/.growpart-wrapped" "$out/bin/growpart" \ + substitute "${pkgs.cloud-utils.guest}/bin/.growpart-wrapped" "$out/bin/growpart" \ --replace "${pkgs.bash}/bin/sh" "/bin/sh" \ --replace "awk" "gawk" \ --replace "sed" "gnused" diff --git a/pkgs/tools/misc/cloud-utils/default.nix b/pkgs/tools/misc/cloud-utils/default.nix index d5c5b0ee783..afae939ef38 100644 --- a/pkgs/tools/misc/cloud-utils/default.nix +++ b/pkgs/tools/misc/cloud-utils/default.nix @@ -1,9 +1,18 @@ { stdenv, fetchurl, makeWrapper , gawk, gnused, utillinux, file , wget, python3, qemu-utils, euca2ools -, e2fsprogs, cdrkit }: +, e2fsprogs, cdrkit +, gptfdisk }: -stdenv.mkDerivation rec { +let + # according to https://packages.debian.org/sid/cloud-image-utils + https://packages.debian.org/sid/admin/cloud-guest-utils + guestDeps = [ + e2fsprogs gptfdisk gawk gnused utillinux + ]; + binDeps = guestDeps ++ [ + wget file qemu-utils cdrkit + ]; +in stdenv.mkDerivation rec { # NOTICE: if you bump this, make sure to run # $ nix-build nixos/release-combined.nix -A nixos.tests.ec2-nixops # growpart is needed in initrd in nixos/system/boot/grow-partition.nix @@ -17,15 +26,24 @@ stdenv.mkDerivation rec { buildInputs = [ python3 ]; installFlags = [ "LIBDIR=$(out)/lib" "BINDIR=$(out)/bin" "MANDIR=$(out)/man/man1" "DOCDIR=$(out)/doc" ]; - # according to https://packages.ubuntu.com/source/zesty/cloud-utils - binDeps = [ - wget e2fsprogs file gnused gawk utillinux qemu-utils euca2ools cdrkit - ]; + # $guest output contains all executables needed for cloud-init and $out the rest + $guest + # This is similar to debian's package split into cloud-image-utils and cloud-guest-utils + # The reason is to reduce the closure size + outputs = [ "out" "guest"]; postFixup = '' + moveToOutput bin/ec2metadata $guest + moveToOutput bin/growpart $guest + moveToOutput bin/vcs-run $guest + for i in $out/bin/*; do wrapProgram $i --prefix PATH : "${stdenv.lib.makeBinPath binDeps}:$out/bin" done + + for i in $guest/bin/*; do + wrapProgram $i --prefix PATH : "${stdenv.lib.makeBinPath guestDeps}:$guest/bin" + ln -s $i $out/bin + done ''; dontBuild = true; From 3ca2fd5520e288afd329e6274a5cc6c0524ec5a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 18 Aug 2020 09:33:47 +0100 Subject: [PATCH 2/5] cloud-init: 0.7.9 -> 20.2 --- nixos/tests/cloud-init.nix | 7 +- ...ort.patch => 0001-add-nixos-support.patch} | 46 +++++++--- .../virtualization/cloud-init/default.nix | 87 ++++++++++++++----- pkgs/top-level/all-packages.nix | 2 +- 4 files changed, 105 insertions(+), 37 deletions(-) rename pkgs/tools/virtualization/cloud-init/{add-nixos-support.patch => 0001-add-nixos-support.patch} (73%) diff --git a/nixos/tests/cloud-init.nix b/nixos/tests/cloud-init.nix index aafa6e24e84..8debb9e8ed9 100644 --- a/nixos/tests/cloud-init.nix +++ b/nixos/tests/cloud-init.nix @@ -24,7 +24,8 @@ let instance-id: iid-local01 local-hostname: "test" public-keys: - - "should be a key!" + ec2-keypair.us-east-1: + - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB5L7Xuh49VS5VQheFE7VDmXKH0BOnB1R0avAE91QgOB root@test EOF ${pkgs.cdrkit}/bin/genisoimage -volid cidata -joliet -rock -o $out/metadata.iso $out/iso ''; @@ -45,8 +46,6 @@ in makeTest { machine.wait_for_unit("cloud-init.service") machine.succeed("cat /tmp/cloudinit-write-file | grep -q 'cloudinit'") - machine.wait_until_succeeds( - "cat /root/.ssh/authorized_keys | grep -q 'should be a key!'" - ) + machine.wait_until_succeeds("cat /root/.ssh/authorized_keys | grep -q root@test") ''; } diff --git a/pkgs/tools/virtualization/cloud-init/add-nixos-support.patch b/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch similarity index 73% rename from pkgs/tools/virtualization/cloud-init/add-nixos-support.patch rename to pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch index 00cb7d070c6..ef8f2b65ca0 100644 --- a/pkgs/tools/virtualization/cloud-init/add-nixos-support.patch +++ b/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch @@ -1,18 +1,36 @@ -diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/__init__.py cloud-init-0.7.6/cloudinit/distros/__init__.py ---- cloud-init-0.7.6.orig/cloudinit/distros/__init__.py 2014-10-10 15:26:25.000000000 +0000 -+++ cloud-init-0.7.6/cloudinit/distros/__init__.py 2016-06-08 07:51:45.230357099 +0000 -@@ -43,6 +43,7 @@ +From 64a767136c16aad2b94b4d9a3268b0d4deba7272 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= +Date: Tue, 18 Aug 2020 10:22:36 +0100 +Subject: [PATCH] add nixos support +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: Jörg Thalheim +--- + cloudinit/distros/__init__.py | 1 + + cloudinit/distros/nixos.py | 103 ++++++++++++++++++++++++++++++++++ + 2 files changed, 104 insertions(+) + create mode 100644 cloudinit/distros/nixos.py + +diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py +index c7163e1c..c147e2b3 100755 +--- a/cloudinit/distros/__init__.py ++++ b/cloudinit/distros/__init__.py +@@ -46,6 +46,7 @@ OSFAMILIES = { 'freebsd': ['freebsd'], - 'suse': ['sles'], + 'suse': ['opensuse', 'sles'], 'arch': ['arch'], + 'nixos': ['nixos'], } LOG = logging.getLogger(__name__) -diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/cloudinit/distros/nixos.py ---- cloud-init-0.7.6.orig/cloudinit/distros/nixos.py 1970-01-01 00:00:00.000000000 +0000 -+++ cloud-init-0.7.6/cloudinit/distros/nixos.py 2016-06-08 07:50:58.602616595 +0000 -@@ -0,0 +1,98 @@ +diff --git a/cloudinit/distros/nixos.py b/cloudinit/distros/nixos.py +new file mode 100644 +index 00000000..d53d2a61 +--- /dev/null ++++ b/cloudinit/distros/nixos.py +@@ -0,0 +1,103 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. @@ -35,10 +53,13 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + ++import os ++ +from cloudinit import distros +from cloudinit import helpers +from cloudinit import log as logging +from cloudinit import util ++from cloudinit import atomic_helper + +from cloudinit.distros.parsers.hostname import HostnameConf + @@ -52,6 +73,8 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) ++ self.usr_lib_exec = os.path.join(os.path.dirname(__file__), ++ "../../../../../libexec") + self.osfamily = 'nixos' + + def _select_hostname(self, hostname, fqdn): @@ -72,7 +95,7 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou + if not conf: + conf = HostnameConf('') + conf.set_hostname(your_hostname) -+ util.write_file(out_fn, str(conf), 0644) ++ atomic_helper.write_file(out_fn, str(conf).encode("utf-8")) + + def _read_system_hostname(self): + sys_hostname = self._read_hostname(self.hostname_conf_fn) @@ -111,3 +134,6 @@ diff -ruN cloud-init-0.7.6.orig/cloudinit/distros/nixos.py cloud-init-0.7.6/clou + + def update_package_sources(self): + raise NotImplementedError() +-- +2.28.0 + diff --git a/pkgs/tools/virtualization/cloud-init/default.nix b/pkgs/tools/virtualization/cloud-init/default.nix index bf932ff1ad4..66ce16d985e 100644 --- a/pkgs/tools/virtualization/cloud-init/default.nix +++ b/pkgs/tools/virtualization/cloud-init/default.nix @@ -1,40 +1,83 @@ -{ lib, pythonPackages, fetchurl, cloud-utils }: +{ lib +, fetchFromGitHub +, buildPythonApplication +, jinja2 +, oauthlib +, configobj +, pyyaml +, requests +, jsonschema +, jsonpatch +, pytest +, httpretty +, dmidecode +, pytestCheckHook +, shadow +, cloud-utils +, openssh +}: -let version = "0.7.9"; +let version = "20.2"; -in pythonPackages.buildPythonApplication { +in buildPythonApplication { pname = "cloud-init"; inherit version; namePrefix = ""; - src = fetchurl { - url = "https://launchpad.net/cloud-init/trunk/${version}/+download/cloud-init-${version}.tar.gz"; - sha256 = "0wnl76pdcj754pl99wxx76hkir1s61x0bg0lh27sdgdxy45vivbn"; + src = fetchFromGitHub { + owner = "canonical"; + repo = "cloud-init"; + rev = version; + sha256 = "sha256-QeY/fdIIUSsp5oNxyRtZwpTB747Jf5KAJuYY9yiKUvc="; }; - patches = [ ./add-nixos-support.patch ]; + patches = [ ./0001-add-nixos-support.patch ]; prePatch = '' - patchShebangs ./tools + substituteInPlace setup.py --replace /lib/systemd $out/lib/systemd + ''; - substituteInPlace setup.py \ - --replace /usr $out \ - --replace /etc $out/etc \ - --replace /lib/systemd $out/lib/systemd \ - --replace 'self.init_system = ""' 'self.init_system = "systemd"' + postInstall = '' + install -D -m755 ./tools/write-ssh-key-fingerprints $out/libexec/write-ssh-key-fingerprints + for i in $out/libexec/*; do + wrapProgram $i --prefix PATH : "${lib.makeBinPath [ openssh ]}" + done + ''; - substituteInPlace cloudinit/config/cc_growpart.py \ - --replace 'util.subp(["growpart"' 'util.subp(["${cloud-utils}/bin/growpart"' + propagatedBuildInputs = [ + jinja2 + oauthlib + configobj + pyyaml + requests + jsonschema + jsonpatch + ]; - # Argparse is part of python stdlib - sed -i s/argparse// requirements.txt - ''; + checkInputs = [ + pytestCheckHook + httpretty + dmidecode + # needed for tests; at runtime we rather want the setuid wrapper + shadow + ]; - propagatedBuildInputs = with pythonPackages; [ cheetah jinja2 prettytable - oauthlib pyserial configobj pyyaml requests jsonpatch ]; + makeWrapperArgs = [ + "--prefix PATH : ${lib.makeBinPath [ + dmidecode cloud-utils.guest + ]}/bin" + ]; - checkInputs = with pythonPackages; [ contextlib2 httpretty mock unittest2 ]; + disabledTests = [ + # tries to create /var + "test_dhclient_run_with_tmpdir" + # clears path and fails because mkdir is not found + "test_path_env_gets_set_from_main" + ]; - doCheck = false; + preCheck = '' + # TestTempUtils.test_mkdtemp_default_non_root does not like TMPDIR=/build + export TMPDIR=/tmp + ''; meta = { homepage = "https://cloudinit.readthedocs.org"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 41e7af4c09d..0ca018ee10b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1557,7 +1557,7 @@ in clog-cli = callPackage ../development/tools/clog-cli { }; - cloud-init = callPackage ../tools/virtualization/cloud-init { }; + cloud-init = python3.pkgs.callPackage ../tools/virtualization/cloud-init { }; cloudflared = callPackage ../applications/networking/cloudflared { }; From 00347e9537d6e6ddef599094429a1ac5392d6f28 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sat, 18 Jan 2020 01:09:49 +0100 Subject: [PATCH 3/5] nixos/cloud-init: actually test ssh'ing, also, with cloud-init-created user --- nixos/tests/cloud-init.nix | 49 ++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/nixos/tests/cloud-init.nix b/nixos/tests/cloud-init.nix index 8debb9e8ed9..d23e9c64b37 100644 --- a/nixos/tests/cloud-init.nix +++ b/nixos/tests/cloud-init.nix @@ -7,6 +7,9 @@ with import ../lib/testing-python.nix { inherit system pkgs; }; with pkgs.lib; let + inherit (import ./ssh-keys.nix pkgs) + snakeOilPrivateKey snakeOilPublicKey; + metadataDrive = pkgs.stdenv.mkDerivation { name = "metadata"; buildCommand = '' @@ -18,14 +21,19 @@ let - content: | cloudinit path: /tmp/cloudinit-write-file + + users: + - default + - name: nixos + ssh_authorized_keys: + - "${snakeOilPublicKey}" EOF cat << EOF > $out/iso/meta-data instance-id: iid-local01 local-hostname: "test" public-keys: - ec2-keypair.us-east-1: - - ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB5L7Xuh49VS5VQheFE7VDmXKH0BOnB1R0avAE91QgOB root@test + - "${snakeOilPublicKey}" EOF ${pkgs.cdrkit}/bin/genisoimage -volid cidata -joliet -rock -o $out/metadata.iso $out/iso ''; @@ -35,17 +43,32 @@ in makeTest { meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ lewo ]; }; - machine = - { ... }: - { - virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ]; - services.cloud-init.enable = true; - }; - testScript = '' - machine.start() - machine.wait_for_unit("cloud-init.service") - machine.succeed("cat /tmp/cloudinit-write-file | grep -q 'cloudinit'") + machine = { ... }: + { + virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ]; + services.cloud-init.enable = true; + services.openssh.enable = true; - machine.wait_until_succeeds("cat /root/.ssh/authorized_keys | grep -q root@test") + }; + testScript = '' + machine.wait_for_unit("cloud-init.service") + machine.succeed("cat /tmp/cloudinit-write-file | grep -q 'cloudinit'") + + # install snakeoil ssh key and provision .ssh/config file + machine.succeed("mkdir -p ~/.ssh") + machine.succeed( + "cat ${snakeOilPrivateKey} > ~/.ssh/id_snakeoil" + ) + machine.succeed("chmod 600 ~/.ssh/id_snakeoil") + + machine.wait_for_unit("sshd.service") + + # we should be able to log in as the root user, as well as the created nixos user + machine.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil root@localhost 'true'" + ) + machine.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'true'" + ) ''; } From a299fc23515d50d8161027af87b649eca27662d8 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Thu, 15 Oct 2020 00:24:39 +0200 Subject: [PATCH 4/5] nixosTests.cloud-init: test changing hostname actually works --- nixos/tests/cloud-init.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/nixos/tests/cloud-init.nix b/nixos/tests/cloud-init.nix index d23e9c64b37..a127be6dd85 100644 --- a/nixos/tests/cloud-init.nix +++ b/nixos/tests/cloud-init.nix @@ -48,7 +48,7 @@ in makeTest { virtualisation.qemu.options = [ "-cdrom" "${metadataDrive}/metadata.iso" ]; services.cloud-init.enable = true; services.openssh.enable = true; - + networking.hostName = ""; }; testScript = '' machine.wait_for_unit("cloud-init.service") @@ -70,5 +70,13 @@ in makeTest { machine.succeed( "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'true'" ) + + # test changing hostname via cloud-init worked + assert ( + machine.succeed( + "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentityFile=~/.ssh/id_snakeoil nixos@localhost 'hostname'" + ).strip() + == "test" + ) ''; } From 831067656061d820fea82f76df76c970366c2191 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Thu, 15 Oct 2020 01:09:00 +0200 Subject: [PATCH 5/5] cloud-init: 20.2 -> 20.3 required rebasing the patch, disabling some tests. I also changed the hash to be in conventional format - the hash mismatch in fixed-output derivation '/nix/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-source': wanted: sha256:yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy got: sha256:zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz doesn't propose SRI syntax. --- .../cloud-init/0001-add-nixos-support.patch | 10 +++++----- pkgs/tools/virtualization/cloud-init/default.nix | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch b/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch index ef8f2b65ca0..997d28aaecb 100644 --- a/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch +++ b/pkgs/tools/virtualization/cloud-init/0001-add-nixos-support.patch @@ -1,4 +1,4 @@ -From 64a767136c16aad2b94b4d9a3268b0d4deba7272 Mon Sep 17 00:00:00 2001 +From 269cc4c9558549f340ec186d9246654564b2f633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 18 Aug 2020 10:22:36 +0100 Subject: [PATCH] add nixos support @@ -14,13 +14,13 @@ Signed-off-by: Jörg Thalheim create mode 100644 cloudinit/distros/nixos.py diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py -index c7163e1c..c147e2b3 100755 +index 2537608f..c533b585 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py -@@ -46,6 +46,7 @@ OSFAMILIES = { - 'freebsd': ['freebsd'], +@@ -47,6 +47,7 @@ OSFAMILIES = { + 'gentoo': ['gentoo'], + 'redhat': ['amazon', 'centos', 'fedora', 'rhel'], 'suse': ['opensuse', 'sles'], - 'arch': ['arch'], + 'nixos': ['nixos'], } diff --git a/pkgs/tools/virtualization/cloud-init/default.nix b/pkgs/tools/virtualization/cloud-init/default.nix index 66ce16d985e..24ecb7f99c3 100644 --- a/pkgs/tools/virtualization/cloud-init/default.nix +++ b/pkgs/tools/virtualization/cloud-init/default.nix @@ -17,7 +17,7 @@ , openssh }: -let version = "20.2"; +let version = "20.3"; in buildPythonApplication { pname = "cloud-init"; @@ -28,7 +28,7 @@ in buildPythonApplication { owner = "canonical"; repo = "cloud-init"; rev = version; - sha256 = "sha256-QeY/fdIIUSsp5oNxyRtZwpTB747Jf5KAJuYY9yiKUvc="; + sha256 = "1fmckxf4q4sxjqs758vw7ca0rnhl9hyq67cqpqzz2v3s1gqzjhm4"; }; patches = [ ./0001-add-nixos-support.patch ]; @@ -72,6 +72,8 @@ in buildPythonApplication { "test_dhclient_run_with_tmpdir" # clears path and fails because mkdir is not found "test_path_env_gets_set_from_main" + # tries to read from /etc/ca-certificates.conf while inside the sandbox + "test_handler_ca_certs" ]; preCheck = ''