Commit graph

75 commits

Author SHA1 Message Date
Graham Christensen 38a394bdee
Merge pull request #102174 from grahamc/ami-root-use-gpt
AMI root partition table: use GPT to support >2T partitions
2020-10-30 16:14:37 -04:00
Graham Christensen d77ddf2a40
nixos.amazonAmi: use legacy+gpt disk images to support partitions >2T 2020-10-30 15:50:25 -04:00
Graham Christensen 74a577b293
create-amis: improve wording around the service name's IAM role
Co-authored-by: Cole Helbling <cole.e.helbling@outlook.com>
2020-10-30 12:40:17 -04:00
Graham Christensen 2bf1fc0345
create-amis: allow customizing the service role name
The complete setup on the AWS end can be configured
with the following Terraform configuration. It generates
a ./credentials.sh which I just copy/pasted in to the
create-amis.sh script near the top. Note: the entire stack
of users and bucket can be destroyed at the end of the
import.

    variable "region" {
      type = string
    }
    variable "availability_zone" {
      type = string
    }

    provider "aws" {
      region = var.region
    }

    resource "aws_s3_bucket" "nixos-amis" {
      bucket_prefix = "nixos-amis-"
      lifecycle_rule {
        enabled = true
        abort_incomplete_multipart_upload_days = 1
        expiration {
          days = 7
        }
      }
    }

    resource "local_file" "credential-file" {
      file_permission = "0700"
      filename = "${path.module}/credentials.sh"
      sensitive_content = <<SCRIPT
    export service_role_name="${aws_iam_role.vmimport.name}"
    export bucket="${aws_s3_bucket.nixos-amis.bucket}"
    export AWS_ACCESS_KEY_ID="${aws_iam_access_key.uploader.id}"
    export AWS_SECRET_ACCESS_KEY="${aws_iam_access_key.uploader.secret}"
    SCRIPT
    }

    # The following resources are for the *uploader*
    resource "aws_iam_user" "uploader" {
      name = "nixos-amis-uploader"
    }

    resource "aws_iam_access_key" "uploader" {
      user = aws_iam_user.uploader.name
    }

    resource "aws_iam_user_policy" "upload-to-nixos-amis" {
      user = aws_iam_user.uploader.name

      policy = data.aws_iam_policy_document.upload-policy-document.json
    }

    data "aws_iam_policy_document" "upload-policy-document" {
      statement {
        effect = "Allow"

        actions = [
          "s3:ListBucket",
          "s3:GetBucketLocation",
        ]

        resources = [
          aws_s3_bucket.nixos-amis.arn
        ]
      }

      statement {
        effect = "Allow"

        actions = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:DeleteObject",
        ]

        resources = [
          "${aws_s3_bucket.nixos-amis.arn}/*"
        ]
      }

      statement {
        effect = "Allow"
        actions = [
          "ec2:ImportSnapshot",
          "ec2:DescribeImportSnapshotTasks",
          "ec2:DescribeImportSnapshotTasks",
          "ec2:RegisterImage",
          "ec2:DescribeImages"
        ]
        resources = [
          "*"
        ]
      }
    }

    # The following resources are for the *vmimport service user*
    # See: https://docs.aws.amazon.com/vm-import/latest/userguide/vmie_prereqs.html#vmimport-role
    resource "aws_iam_role" "vmimport" {
      assume_role_policy = data.aws_iam_policy_document.vmimport-trust.json
    }

    resource "aws_iam_role_policy" "vmimport-access" {
      role = aws_iam_role.vmimport.id
      policy = data.aws_iam_policy_document.vmimport-access.json
    }

    data "aws_iam_policy_document" "vmimport-access" {
      statement {
        effect = "Allow"
        actions = [
          "s3:GetBucketLocation",
          "s3:GetObject",
          "s3:ListBucket",
        ]
        resources = [
          aws_s3_bucket.nixos-amis.arn,
          "${aws_s3_bucket.nixos-amis.arn}/*"
        ]
      }
      statement {
        effect = "Allow"
        actions = [
          "ec2:ModifySnapshotAttribute",
          "ec2:CopySnapshot",
          "ec2:RegisterImage",
          "ec2:Describe*"
        ]
        resources = [
          "*"
        ]
      }
    }

    data "aws_iam_policy_document" "vmimport-trust" {
      statement {
        effect = "Allow"
        principals {
          type = "Service"
          identifiers = [ "vmie.amazonaws.com" ]
        }

        actions = [
          "sts:AssumeRole"
        ]

        condition {
          test = "StringEquals"
          variable = "sts:ExternalId"
          values = [ "vmimport" ]
        }
      }
    }
2020-10-30 12:12:08 -04:00
Graham Christensen e253de8a77
create-amis.sh: log the full response if describing the import snapshot tasks fails 2020-10-30 12:08:01 -04:00
Graham Christensen f92a883ddb
nixos ec2/create-amis.sh: shellcheck: $ is not needed in arithmetic 2020-10-30 12:08:01 -04:00
Graham Christensen 7dac8470cf
nixos ec2/create-amis.sh: shellcheck: explicitly make the additions to block_device_mappings single strings 2020-10-30 12:08:00 -04:00
Graham Christensen a66a22ca54
nixos ec2/create-amis.sh: shellcheck: read without -r mangles backslashes 2020-10-30 12:08:00 -04:00
Graham Christensen baf7ed3f24
nixos ec2/create-amis.sh: shellcheck: SC2155: Declare and assign separately to avoid masking return values. 2020-10-30 12:07:59 -04:00
Graham Christensen f5994c208d
nixos ec2/create-amis.sh: shellcheck: quote state_dir reference 2020-10-30 12:07:59 -04:00
Graham Christensen c76692192a
nixos ec2/create-amis.sh: shellcheck: quote region references 2020-10-30 12:07:49 -04:00
Lassulus d08b81c3b5
Merge pull request #89116 from wagdav/fix-args-create-amis
nixos/maintainers/scripts/ec2/create-amis.sh: fix argument check
2020-08-22 16:47:54 +02:00
Jörg Thalheim 0cb79c953d nixos/ec2: remove dependency on NIX_PATH
This is required when migrating to flakes
2020-08-16 10:56:44 +00:00
David Wagner 3b1ed035c3 create-amis: fix argument check
Because this script enables `set -u` when no arguments are provided bash
exits with the error:

    $1: unbound variable

instead of the helpful usage message.
2020-05-28 17:41:45 +02:00
Benjamin Hipple 129176452c nixos-ami: update nvme_core.io_timeout for linux kernel >= 4.15
NixOS 20.03 is built on kernel 5.4 and 19.09 is on 4.19, so we should update
this option to the highest value possible, per linked upstream instructions from
Amazon.
2020-03-22 00:35:56 -04:00
adisbladis 4e5b0571ed
create-amis: Add eu-north-1 2020-03-05 18:00:28 +00:00
Andrew Childs bd61216f55 ec2/create-amis.sh: register root device as /dev/xvda
For the case of blkfront drives, there appears to be no difference
between /dev/sda1 and /dev/xvda: the drive always appears as the
kernel device /dev/xvda.

For the case of nvme drives, the root device typically appears as
/dev/nvme0n1.  Amazon provides the 'ec2-utils' package for their first
party linux ("Amazon Linux"), which configures udev to create symlinks
from the provided name to the nvme device name. This name is
communicated through nvme "Identify Controller" response, which can be
inspected with:

  nvme id-ctrl --raw-binary /dev/nvme0n1 | cut -c3073-3104 | hexdump -C

On Amazon Linux, where the device is attached as "/dev/xvda", this
creates:

- /dev/xvda  -> nvme0n1
- /dev/xvda1 -> nvme0n1p1

On NixOS where the device is attach as "/dev/sda1", this creates:

- /dev/sda1  -> nvme0n1
- /dev/sda11 -> nvme0n1p1

This is odd, but not inherently a problem.

NixOS unconditionally configures grub to install to `/dev/xvda`, which
fails on an instance using nvme storage. With the root device name set
to xvda, both blkfront and nvme drives are accessible as /dev/xvda,
either directly or by symlink.
2019-11-02 05:58:58 +09:00
AmineChikhaoui dc13a7f26a
ec2-amis.nix: add 19.09 amis
replace /home/deploy -> $HOME to allow running the script from outside
the bastion.
2019-10-28 14:04:20 -04:00
Andrew Childs 84742e2293 amazon-image.nix: upload prebuilt images 2019-09-05 00:52:21 +09:00
Andrew Childs 25bee972e9 amazon-image.nix: add hydra-build-products and improve metadata 2019-09-05 00:52:20 +09:00
Andrew Childs 027e5560bd amazon-image.nix: default to vpc formatted images
These can be imported without converison.
2019-09-05 00:52:20 +09:00
Andrew Childs 5501274b5f amazon-image.nix: add EFI support, enable by default for aarch64 2019-09-05 00:52:17 +09:00
talyz 261372b69c amazon-image.nix: Resolve failure to include resize2fs
Since 34234dcb51, for resize2fs to be automatically included in
initrd, a filesystem needed for boot must be explicitly defined as an
ext* type filesystem.
2019-03-15 17:33:45 +01:00
Eelco Dolstra b240822cfa create-amis.sh: Change directory for AMIs 2018-07-24 21:19:14 +02:00
Ihor Antonov 08ebd830a5 Fix kernel crash caused by absent root device 2018-05-12 02:55:33 +03:00
Ihor Antonov e4777ae2d8 Fix kernel panic on ec2 kvm instances caused by io timeout on nvme root volume 2018-05-12 02:55:33 +03:00
Maximilian Bosch 9274ea3903
treewide: rename version attributes
As suggested in https://github.com/NixOS/nixpkgs/pull/39416#discussion_r183845745
the versioning attributes in `lib` should be consistent to
`nixos/version` which implicates the following changes:

* `lib.trivial.version` -> `lib.trivial.release`
* `lib.trivial.suffix` -> `lib.trivial.versionSuffix`
* `lib.nixpkgsVersion` -> `lib.version`

As `lib.nixpkgsVersion` is referenced several times in `NixOS/nixpkgs`,
`NixOS/nix` and probably several user's setups. As the rename will cause
a notable impact it's better to keep `lib.nixpkgsVersion` as alias with
a warning yielded by `builtins.trace`.
2018-04-28 14:23:53 +02:00
Tuomas Tynkkynen 962e79ef32 nixos/make-disk-image.nix: Support EFI images
- Add a new parameter `imageType` that can specify either "efi" or
  "legacy" (the default which should see no change in behaviour by
  this patch).

- EFI images get a GPT partition table (instead of msdos) with a
  mandatory ESP partition (so we add an assert that `partitioned`
  is true).

- Use the partx tool from util-linux to determine exact start + size
  of the root partition. This is required because GPT stores a secondary
  partition table at the end of the disk, so we can't just have
  mkfs.ext4 create the filesystem until the end of the disk.

- (Unrelated to any EFI changes) Since we're depending on the
  `-E offset=X` option to mkfs which is only supported by e2fsprogs,
  disallow any attempts of creating partitioned disk images where
  the root filesystem is not ext4.
2018-01-22 11:18:23 +02:00
Eelco Dolstra 014800706a
create-amis.sh: Ass eu-west-3 2017-12-20 16:35:22 +01:00
Dan Peebles 63f7456f6a amazon-image: make derivation name configurable
I'm getting dozens of nixos-disk-image entries in my store and it's
getting hard to track :)
2017-10-23 16:22:24 -04:00
Eelco Dolstra 6c72efe0ba
Don't generate instance-store AMIs
These are obsolete, use EBS AMIs instead.
2017-09-28 17:33:13 +02:00
Dan Peebles ca99ec3f58 amazon-image: fix typo in last commit 2017-08-11 02:12:39 +00:00
Dan Peebles 5def5bab3c amazon-image: make image size configurable 2017-08-11 01:57:43 +00:00
Dan Peebles 5f372ef67f amazon-image: support VPC/VHD as an output format for the builder 2017-08-10 22:57:26 +00:00
Dan Peebles f3f8c584a4 amazon-image: Add disk format option to image builder module 2017-08-10 21:40:42 +00:00
Dan Peebles f1708a9d7d make-disk-image: change to be less VM-centric
This changes much of the make-disk-image.nix logic (and thus most NixOS
image building) to use LKL to set up the target directory structure rather
than a Linux VM. The only work we still do in a VM is less IO-heavy stuff
that while still time-consuming, is less of the overall load. The goal is
to kill more of that stuff, but that will require deeper changes to NixOS
activation scripts and switch-to-configuration.pl, and I don't want to
bite off too much at once.
2017-04-24 02:30:00 +00:00
Eelco Dolstra 279565c3d6
Revert "Revert "EC2: Disable PV support""
This reverts commit 71710fd099.
2017-04-04 13:03:05 +02:00
Jörg Thalheim 71710fd099
Revert "EC2: Disable PV support"
This reverts commit fbe6d23624.

this breaks every non-ec2 (non-hvm) system

cc @edolstra
2017-04-04 12:05:21 +02:00
Eelco Dolstra fbe6d23624
EC2: Disable PV support
Unfortunately, somewhere between 16.09 and 17.03, paravirtualized
instances stopped working. They hang at the pv-grub prompt
("grubdom>"). I tried reverting to a 4.4 kernel, reverting kernel
compression from xz to bzip2 (even though pv-grub is supposed to
support xz), and reverting the only change to initrd generation
(5a8147479e). Nothing worked so I'm
giving up.
2017-04-03 17:46:34 +02:00
Eelco Dolstra e6faf2a4e6
create-amis.sh: Use pv-grub-hd0_1.05 2017-04-03 17:46:34 +02:00
Dan Peebles 49641e0de0 make-disk-image.nix: support additional filesystem contents
This makes make-disk-image.nix slightly more consistent with other image
builders we have. Unfortunately I duplicated some code in doing so, but
this is temporary duplication on the path to consolidating everything.
See https://github.com/NixOS/nixpkgs/issues/23052 for more details on that.

I'm also exposing the option in the amazon-image.nix maintainer module.
2017-02-22 23:49:49 +00:00
Eelco Dolstra a2b8ceb83a Create AMIs for ca-central-1 (Canada) 2017-02-19 23:19:07 +01:00
Eelco Dolstra 4e516363a8 Create AMIs for eu-west-2 (London) 2017-01-17 21:44:01 +01:00
Domen Kožar 49d608ac00 create-amis: use jq instead of json 2016-11-22 01:59:49 +01:00
Domen Kožar f940d65b2d create-amis: add us-east-2 2016-11-21 21:26:23 +01:00
Domen Kožar 1944c984c3 create-amis: order matters 2016-11-21 16:43:09 +01:00
Domen Kožar 6e08a55474 create-amis.sh: another dep needed for EBS images 2016-11-21 15:56:51 +01:00
Domen Kožar 67f3e2853b create-amis.sh: use nix-shell for convenience 2016-11-16 16:49:32 +01:00
Eelco Dolstra 94cc18e9aa Add AMIs in ap-northeast-2 and ap-south-1 2016-07-12 17:26:25 +02:00
Eelco Dolstra 1e9b8bfb31 Copy AMIs in parallel 2016-07-12 17:26:16 +02:00