diff options
Diffstat (limited to 'nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh')
| -rwxr-xr-x | nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh | 119 |
1 files changed, 74 insertions, 45 deletions
diff --git a/nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh b/nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh index 691d7fcfcba4..712d9b548ff0 100755 --- a/nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh +++ b/nixpkgs/nixos/maintainers/scripts/ec2/create-amis.sh @@ -15,18 +15,22 @@ # set -x set -euo pipefail +var () { true; } + # configuration -state_dir=$HOME/amis/ec2-images -home_region=eu-west-1 -bucket=nixos-amis -service_role_name=vmimport +var ${state_dir:=$HOME/amis/ec2-images} +var ${home_region:=eu-west-1} +var ${bucket:=nixos-amis} +var ${service_role_name:=vmimport} -regions=(eu-west-1 eu-west-2 eu-west-3 eu-central-1 eu-north-1 +var ${regions:=eu-west-1 eu-west-2 eu-west-3 eu-central-1 eu-north-1 us-east-1 us-east-2 us-west-1 us-west-2 ca-central-1 ap-southeast-1 ap-southeast-2 ap-northeast-1 ap-northeast-2 ap-south-1 ap-east-1 - sa-east-1) + sa-east-1} + +regions=($regions) log() { echo "$@" >&2 @@ -60,10 +64,16 @@ read_image_info() { # We handle a single image per invocation, store all attributes in # globals for convenience. -image_label=$(read_image_info .label) +zfs_disks=$(read_image_info .disks) +image_label="$(read_image_info .label)${zfs_disks:+-ZFS}" image_system=$(read_image_info .system) -image_file=$(read_image_info .file) -image_logical_bytes=$(read_image_info .logical_bytes) +image_files=( $(read_image_info "${zfs_disks:+.disks.root}.file") ) + +image_logical_bytes=$(read_image_info "${zfs_disks:+.disks.boot}.logical_bytes") + +if [[ -n "$zfs_disks" ]]; then + image_files+=( $(read_image_info .disks.boot.file) ) +fi # Derived attributes @@ -113,11 +123,11 @@ wait_for_import() { local state snapshot_id log "Waiting for import task $task_id to be completed" while true; do - read -r state progress snapshot_id < <( + read -r state message snapshot_id < <( aws ec2 describe-import-snapshot-tasks --region "$region" --import-task-ids "$task_id" | \ - jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail | "\(.Status) \(.Progress) \(.SnapshotId)"' + jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail | "\(.Status) \(.StatusMessage) \(.SnapshotId)"' ) - log " ... state=$state progress=$progress snapshot_id=$snapshot_id" + log " ... state=$state message=$message snapshot_id=$snapshot_id" case "$state" in active) sleep 10 @@ -179,41 +189,48 @@ make_image_public() { upload_image() { local region=$1 - local aws_path=${image_file#/} + for image_file in "${image_files[@]}"; do + local aws_path=${image_file#/} - local state_key="$region.$image_label.$image_system" - local task_id - task_id=$(read_state "$state_key" task_id) - local snapshot_id - snapshot_id=$(read_state "$state_key" snapshot_id) - local ami_id - ami_id=$(read_state "$state_key" ami_id) - - if [ -z "$task_id" ]; then - log "Checking for image on S3" - if ! aws s3 ls --region "$region" "s3://${bucket}/${aws_path}" >&2; then - log "Image missing from aws, uploading" - aws s3 cp --region "$region" "$image_file" "s3://${bucket}/${aws_path}" >&2 + if [[ -n "$zfs_disks" ]]; then + local suffix=${image_file%.*} + suffix=${suffix##*.} fi - log "Importing image from S3 path s3://$bucket/$aws_path" - - task_id=$(aws ec2 import-snapshot --role-name "$service_role_name" --disk-container "{ - \"Description\": \"nixos-image-${image_label}-${image_system}\", - \"Format\": \"vhd\", - \"UserBucket\": { - \"S3Bucket\": \"$bucket\", - \"S3Key\": \"$aws_path\" - } - }" --region "$region" | jq -r '.ImportTaskId') - - write_state "$state_key" task_id "$task_id" - fi + local state_key="$region.$image_label${suffix:+.${suffix}}.$image_system" + local task_id + task_id=$(read_state "$state_key" task_id) + local snapshot_id + snapshot_id=$(read_state "$state_key" snapshot_id) + local ami_id + ami_id=$(read_state "$state_key" ami_id) + + if [ -z "$task_id" ]; then + log "Checking for image on S3" + if ! aws s3 ls --region "$region" "s3://${bucket}/${aws_path}" >&2; then + log "Image missing from aws, uploading" + aws s3 cp --region "$region" "$image_file" "s3://${bucket}/${aws_path}" >&2 + fi + + log "Importing image from S3 path s3://$bucket/$aws_path" + + task_id=$(aws ec2 import-snapshot --role-name "$service_role_name" --disk-container "{ + \"Description\": \"nixos-image-${image_label}-${image_system}\", + \"Format\": \"vhd\", + \"UserBucket\": { + \"S3Bucket\": \"$bucket\", + \"S3Key\": \"$aws_path\" + } + }" --region "$region" | jq -r '.ImportTaskId') + + write_state "$state_key" task_id "$task_id" + fi - if [ -z "$snapshot_id" ]; then - snapshot_id=$(wait_for_import "$region" "$task_id") - write_state "$state_key" snapshot_id "$snapshot_id" - fi + if [ -z "$snapshot_id" ]; then + snapshot_id=$(wait_for_import "$region" "$task_id") + write_state "$state_key" snapshot_id "$snapshot_id" + fi + done if [ -z "$ami_id" ]; then log "Registering snapshot $snapshot_id as AMI" @@ -222,6 +239,18 @@ upload_image() { "DeviceName=/dev/xvda,Ebs={SnapshotId=$snapshot_id,VolumeSize=$image_logical_gigabytes,DeleteOnTermination=true,VolumeType=gp3}" ) + if [[ -n "$zfs_disks" ]]; then + local root_snapshot_id=$(read_state "$region.$image_label.root.$image_system" snapshot_id) + + local root_image_logical_bytes=$(read_image_info ".disks.root.logical_bytes") + local root_image_logical_gigabytes=$(((root_image_logical_bytes-1)/1024/1024/1024+1)) # Round to the next GB + + block_device_mappings+=( + "DeviceName=/dev/xvdb,Ebs={SnapshotId=$root_snapshot_id,VolumeSize=$root_image_logical_gigabytes,DeleteOnTermination=true,VolumeType=gp3}" + ) + fi + + local extra_flags=( --root-device-name /dev/xvda --sriov-net-support simple @@ -248,7 +277,7 @@ upload_image() { write_state "$state_key" ami_id "$ami_id" fi - make_image_public "$region" "$ami_id" + [[ -v PRIVATE ]] || make_image_public "$region" "$ami_id" echo "$ami_id" } @@ -276,7 +305,7 @@ copy_to_region() { write_state "$state_key" ami_id "$ami_id" fi - make_image_public "$region" "$ami_id" + [[ -v PRIVATE ]] || make_image_public "$region" "$ami_id" echo "$ami_id" } |