From f1708a9d7d79e2bf2961fc648625578b23b3460f Mon Sep 17 00:00:00 2001 From: Dan Peebles Date: Mon, 17 Apr 2017 03:08:37 +0000 Subject: [PATCH] make-disk-image: change to be less VM-centric This changes much of the make-disk-image.nix logic (and thus most NixOS image building) to use LKL to set up the target directory structure rather than a Linux VM. The only work we still do in a VM is less IO-heavy stuff that while still time-consuming, is less of the overall load. The goal is to kill more of that stuff, but that will require deeper changes to NixOS activation scripts and switch-to-configuration.pl, and I don't want to bite off too much at once. --- nixos/lib/make-disk-image.nix | 204 +++++++++++------- .../maintainers/scripts/ec2/amazon-image.nix | 5 +- .../installer/tools/nixos-prepare-root.sh | 2 +- 3 files changed, 125 insertions(+), 86 deletions(-) diff --git a/nixos/lib/make-disk-image.nix b/nixos/lib/make-disk-image.nix index 8c5de22f30f..56766ec9047 100644 --- a/nixos/lib/make-disk-image.nix +++ b/nixos/lib/make-disk-image.nix @@ -33,42 +33,124 @@ , name ? "nixos-disk-image" - # This prevents errors while checking nix-store validity, see - # https://github.com/NixOS/nix/issues/1134 -, fixValidity ? true - , format ? "raw" }: with lib; -pkgs.vmTools.runInLinuxVM ( +let + # Copied from https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/installer/cd-dvd/channel.nix + # TODO: factor out more cleanly + + # Do not include these things: + # - The '.git' directory + # - Result symlinks from nix-build ('result', 'result-2', 'result-bin', ...) + # - VIM/Emacs swap/backup files ('.swp', '.swo', '.foo.swp', 'foo~', ...) + filterFn = path: type: let basename = baseNameOf (toString path); in + if type == "directory" then basename != ".git" + else if type == "symlink" then builtins.match "^result(|-.*)$" basename == null + else builtins.match "^((|\..*)\.sw[a-z]|.*~)$" basename == null; + + nixpkgs = builtins.filterSource filterFn pkgs.path; + + channelSources = pkgs.runCommand "nixos-${config.system.nixosVersion}" {} '' + mkdir -p $out + cp -prd ${nixpkgs} $out/nixos + chmod -R u+w $out/nixos + if [ ! -e $out/nixos/nixpkgs ]; then + ln -s . $out/nixos/nixpkgs + fi + rm -rf $out/nixos/.git + echo -n ${config.system.nixosVersionSuffix} > $out/nixos/.version-suffix + ''; + + metaClosure = pkgs.writeText "meta" '' + ${config.system.build.toplevel} + ${config.nix.package.out} + ${channelSources} + ''; + + prepareImageInputs = with pkgs; [ rsync utillinux parted e2fsprogs lkl fakeroot config.system.build.nixos-prepare-root ] ++ stdenv.initialPath; + + # I'm preserving the line below because I'm going to search for it across nixpkgs to consolidate + # image building logic. The comment right below this now appears in 4 different places in nixpkgs :) + # !!! should use XML. + sources = map (x: x.source) contents; + targets = map (x: x.target) contents; + + prepareImage = '' + export PATH=${pkgs.lib.makeSearchPathOutput "bin" "bin" prepareImageInputs} + + mkdir $out + diskImage=nixos.raw + truncate -s ${toString diskSize}M $diskImage + + ${if partitioned then '' + parted $diskImage -- mklabel msdos mkpart primary ext4 1M -1s + offset=$((2048*512)) + '' else '' + offset=0 + ''} + + mkfs.${fsType} -F -L nixos -E offset=$offset $diskImage + + root="$PWD/root" + mkdir -p $root + + # Copy arbitrary other files into the image + # Semi-shamelessly copied from make-etc.sh. I (@copumpkin) shall factor this stuff out as part of + # https://github.com/NixOS/nixpkgs/issues/23052. + set -f + sources_=(${concatStringsSep " " sources}) + targets_=(${concatStringsSep " " targets}) + set +f + + for ((i = 0; i < ''${#targets_[@]}; i++)); do + source="''${sources_[$i]}" + target="''${targets_[$i]}" + + if [[ "$source" =~ '*' ]]; then + # If the source name contains '*', perform globbing. + mkdir -p $root/$target + for fn in $source; do + rsync -a --no-o --no-g "$fn" $root/$target/ + done + else + mkdir -p $root/$(dirname $target) + if ! [ -e $root/$target ]; then + rsync -a --no-o --no-g $source $root/$target + else + echo "duplicate entry $target -> $source" + exit 1 + fi + fi + done + + # TODO: Nix really likes to chown things it creates to its current user... + fakeroot nixos-prepare-root $root ${channelSources} ${config.system.build.toplevel} closure + + echo "copying staging root to image..." + cptofs ${pkgs.lib.optionalString partitioned "-P 1"} -t ${fsType} -i $diskImage $root/* / + ''; +in pkgs.vmTools.runInLinuxVM ( pkgs.runCommand name - { preVM = - '' - mkdir $out - diskImage=$out/nixos.${if format == "qcow2" then "qcow2" else "img"} - ${pkgs.vmTools.qemu}/bin/qemu-img create -f ${format} $diskImage "${toString diskSize}M" - mv closure xchg/ - ''; - buildInputs = with pkgs; [ utillinux perl e2fsprogs parted rsync ]; - - # I'm preserving the line below because I'm going to search for it across nixpkgs to consolidate - # image building logic. The comment right below this now appears in 4 different places in nixpkgs :) - # !!! should use XML. - sources = map (x: x.source) contents; - targets = map (x: x.target) contents; - - exportReferencesGraph = - [ "closure" config.system.build.toplevel ]; - inherit postVM; + { preVM = prepareImage; + buildInputs = with pkgs; [ utillinux e2fsprogs ]; + exportReferencesGraph = [ "closure" metaClosure ]; + postVM = '' + ${if format == "raw" then '' + mv $diskImage $out/nixos.img + diskImage=$out/nixos.img + '' else '' + ${pkgs.qemu}/bin/qemu-img convert -f raw -O qcow2 $diskImage $out/nixos.qcow2 + diskImage=$out/nixos.qcow2 + ''} + ${postVM} + ''; memSize = 1024; } '' ${if partitioned then '' - # Create a single / partition. - parted /dev/vda mklabel msdos - parted /dev/vda -- mkpart primary ext2 1M -1s . /sys/class/block/vda1/uevent mknod /dev/vda1 b $MAJOR $MINOR rootDisk=/dev/vda1 @@ -76,74 +158,34 @@ pkgs.vmTools.runInLinuxVM ( rootDisk=/dev/vda ''} - # Create an empty filesystem and mount it. - mkfs.${fsType} -L nixos $rootDisk - mkdir /mnt - mount $rootDisk /mnt - - # Register the paths in the Nix database. - printRegistration=1 perl ${pkgs.pathsFromGraph} /tmp/xchg/closure | \ - ${config.nix.package.out}/bin/nix-store --load-db --option build-users-group "" - - ${if fixValidity then '' - # Add missing size/hash fields to the database. FIXME: - # exportReferencesGraph should provide these directly. - ${config.nix.package.out}/bin/nix-store --verify --check-contents --option build-users-group "" - '' else ""} - - # In case the bootloader tries to write to /dev/sda… + # Some tools assume these exist ln -s vda /dev/xvda ln -s vda /dev/sda - # Install the closure onto the image - USER=root ${config.system.build.nixos-install}/bin/nixos-install \ - --closure ${config.system.build.toplevel} \ - --no-channel-copy \ - --no-root-passwd \ - ${optionalString (!installBootLoader) "--no-bootloader"} + mountPoint=/mnt + mkdir $mountPoint + mount $rootDisk $mountPoint - # Install a configuration.nix. + # Install a configuration.nix mkdir -p /mnt/etc/nixos ${optionalString (configFile != null) '' cp ${configFile} /mnt/etc/nixos/configuration.nix ''} - # Remove /etc/machine-id so that each machine cloning this image will get its own id - rm -f /mnt/etc/machine-id + mount --rbind /dev $mountPoint/dev + mount --rbind /proc $mountPoint/proc + mount --rbind /sys $mountPoint/sys - # Copy arbitrary other files into the image - # Semi-shamelessly copied from make-etc.sh. I (@copumpkin) shall factor this stuff out as part of - # https://github.com/NixOS/nixpkgs/issues/23052. - set -f - sources_=($sources) - targets_=($targets) - set +f + # Set up core system link, GRUB, etc. + NIXOS_INSTALL_BOOTLOADER=1 chroot $mountPoint /nix/var/nix/profiles/system/bin/switch-to-configuration boot - for ((i = 0; i < ''${#targets_[@]}; i++)); do - source="''${sources_[$i]}" - target="''${targets_[$i]}" + # TODO: figure out if I should activate, but for now I won't + # chroot $mountPoint /nix/var/nix/profiles/system/activate - if [[ "$source" =~ '*' ]]; then + # The above scripts will generate a random machine-id and we don't want to bake a single ID into all our images + rm -f $mountPoint/etc/machine-id - # If the source name contains '*', perform globbing. - mkdir -p /mnt/$target - for fn in $source; do - rsync -a --no-o --no-g "$fn" /mnt/$target/ - done - - else - - mkdir -p /mnt/$(dirname $target) - if ! [ -e /mnt/$target ]; then - rsync -a --no-o --no-g $source /mnt/$target - else - echo "duplicate entry $target -> $source" - exit 1 - fi - fi - done - - umount /mnt + umount -R /mnt # Make sure resize2fs works. Note that resize2fs has stricter criteria for resizing than a normal # mount, so the `-c 0` and `-i 0` don't affect it. Setting it to `now` doesn't produce deterministic diff --git a/nixos/maintainers/scripts/ec2/amazon-image.nix b/nixos/maintainers/scripts/ec2/amazon-image.nix index b4190df8335..cdfac71634d 100644 --- a/nixos/maintainers/scripts/ec2/amazon-image.nix +++ b/nixos/maintainers/scripts/ec2/amazon-image.nix @@ -6,10 +6,7 @@ let cfg = config.amazonImage; in { - imports = - [ ../../../modules/installer/cd-dvd/channel.nix - ../../../modules/virtualisation/amazon-image.nix - ]; + imports = [ ../../../modules/virtualisation/amazon-image.nix ]; options.amazonImage = { contents = mkOption { diff --git a/nixos/modules/installer/tools/nixos-prepare-root.sh b/nixos/modules/installer/tools/nixos-prepare-root.sh index c374330f846..cd786c47ef6 100644 --- a/nixos/modules/installer/tools/nixos-prepare-root.sh +++ b/nixos/modules/installer/tools/nixos-prepare-root.sh @@ -70,7 +70,7 @@ for i in $closures; do rsync -a $j $mountPoint/nix/store/ done - nix-store --register-validity < $i + nix-store --option build-users-group root --register-validity < $i fi done