From aa46904490cadc55817ec32695c484026ddf3e15 Mon Sep 17 00:00:00 2001 From: Arnold Krille Date: Sat, 30 Jan 2016 23:00:39 +0100 Subject: containers: Add a hostbridge and ipv6 addresses This allows the containers to have their interface in a bridge on the host. Also this adds IPv6 addresses to the containers both with bridged and unbridged network. --- nixos/modules/virtualisation/containers.nix | 100 ++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 19 deletions(-) (limited to 'nixos/modules') diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 121ecbc9bf2c..4c20ee27de2c 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -28,14 +28,23 @@ let # Initialise the container side of the veth pair. if [ "$PRIVATE_NETWORK" = 1 ]; then + ip link set host0 name eth0 ip link set dev eth0 up + + if [ -n "$LOCAL_ADDRESS" ]; then + ip addr add $LOCAL_ADDRESS dev eth0 + fi + if [ -n "$LOCAL_ADDRESS6" ]; then + ip -6 addr add $LOCAL_ADDRESS6 dev eth0 + fi if [ -n "$HOST_ADDRESS" ]; then ip route add $HOST_ADDRESS dev eth0 ip route add default via $HOST_ADDRESS fi - if [ -n "$LOCAL_ADDRESS" ]; then - ip addr add $LOCAL_ADDRESS dev eth0 + if [ -n "$HOST_ADDRESS6" ]; then + ip -6 route add $HOST_ADDRESS6 dev eth0 + ip -6 route add default via $HOST_ADDRESS6 fi fi @@ -48,7 +57,7 @@ let system = config.nixpkgs.system; bindMountOpts = { name, config, ... }: { - + options = { mountPoint = mkOption { example = "/mnt/usb"; @@ -68,13 +77,13 @@ let description = "Determine whether the mounted path will be accessed in read-only mode."; }; }; - + config = { mountPoint = mkDefault name; }; - + }; - + mkBindFlag = d: let flagPrefix = if d.isReadOnly then " --bind-ro=" else " --bind="; mountstr = if d.hostPath != null then "${d.hostPath}:${d.mountPoint}" else "${d.mountPoint}"; @@ -142,12 +151,33 @@ in ''; }; + hostBridge = mkOption { + type = types.nullOr types.string; + default = null; + example = "br0"; + description = '' + Put the host-side of the veth-pair into the named bridge. + Only one of hostAddress* or hostBridge can be given. + ''; + }; + hostAddress = mkOption { type = types.nullOr types.str; default = null; example = "10.231.136.1"; description = '' The IPv4 address assigned to the host interface. + (Not used when hostBridge is set.) + ''; + }; + + hostAddress6 = mkOption { + type = types.nullOr types.string; + default = null; + example = "fc00::1"; + description = '' + The IPv6 address assigned to the host interface. + (Not used when hostBridge is set.) ''; }; @@ -161,6 +191,16 @@ in ''; }; + localAddress6 = mkOption { + type = types.nullOr types.string; + default = null; + example = "fc00::2"; + description = '' + The IPv6 address assigned to eth0 + in the container. + ''; + }; + interfaces = mkOption { type = types.listOf types.string; default = []; @@ -185,7 +225,7 @@ in example = { "/home" = { hostPath = "/home/alice"; isReadOnly = false; }; }; - + description = '' An extra list of directories that is bound to the container. @@ -257,11 +297,7 @@ in if [ "$PRIVATE_NETWORK" = 1 ]; then ip link del dev "ve-$INSTANCE" 2> /dev/null || true - fi - - - if [ "$PRIVATE_NETWORK" = 1 ]; then - ip link del dev "ve-$INSTANCE" 2> /dev/null || true + ip link del dev "vb-$INSTANCE" 2> /dev/null || true fi ''; @@ -281,6 +317,9 @@ in if [ "$PRIVATE_NETWORK" = 1 ]; then extraFlags+=" --network-veth" + if [ -n "$HOST_BRIDGE" ]; then + extraFlags+=" --network-bridge=$HOST_BRIDGE" + fi fi for iface in $INTERFACES; do @@ -315,8 +354,11 @@ in --bind="/nix/var/nix/profiles/per-container/$INSTANCE:/nix/var/nix/profiles" \ --bind="/nix/var/nix/gcroots/per-container/$INSTANCE:/nix/var/nix/gcroots" \ --setenv PRIVATE_NETWORK="$PRIVATE_NETWORK" \ + --setenv HOST_BRIDGE="$HOST_BRIDGE" \ --setenv HOST_ADDRESS="$HOST_ADDRESS" \ --setenv LOCAL_ADDRESS="$LOCAL_ADDRESS" \ + --setenv HOST_ADDRESS6="$HOST_ADDRESS6" \ + --setenv LOCAL_ADDRESS6="$LOCAL_ADDRESS6" \ --setenv PATH="$PATH" \ ${containerInit} "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/init" ''; @@ -324,13 +366,21 @@ in postStart = '' if [ "$PRIVATE_NETWORK" = 1 ]; then - ifaceHost=ve-$INSTANCE - ip link set dev $ifaceHost up - if [ -n "$HOST_ADDRESS" ]; then - ip addr add $HOST_ADDRESS dev $ifaceHost - fi - if [ -n "$LOCAL_ADDRESS" ]; then - ip route add $LOCAL_ADDRESS dev $ifaceHost + if [ -z "$HOST_BRIDGE" ]; then + ifaceHost=ve-$INSTANCE + ip link set dev $ifaceHost up + if [ -n "$HOST_ADDRESS" ]; then + ip addr add $HOST_ADDRESS dev $ifaceHost + fi + if [ -n "$HOST_ADDRESS6" ]; then + ip -6 addr add $HOST_ADDRESS6 dev $ifaceHost + fi + if [ -n "$LOCAL_ADDRESS" ]; then + ip route add $LOCAL_ADDRESS dev $ifaceHost + fi + if [ -n "$LOCAL_ADDRESS6" ]; then + ip -6 route add $LOCAL_ADDRESS6 dev $ifaceHost + fi fi fi @@ -353,6 +403,9 @@ in restartIfChanged = false; #reloadIfChanged = true; # FIXME + wants = [ "netwprk.target" ]; + after = [ "network.target" ]; + serviceConfig = { ExecReload = pkgs.writeScript "reload-container" '' @@ -396,12 +449,21 @@ in SYSTEM_PATH=${cfg.path} ${optionalString cfg.privateNetwork '' PRIVATE_NETWORK=1 + ${optionalString (cfg.hostBridge != null) '' + HOST_BRIDGE=${cfg.hostBridge} + ''} ${optionalString (cfg.hostAddress != null) '' HOST_ADDRESS=${cfg.hostAddress} ''} + ${optionalString (cfg.hostAddress6 != null) '' + HOST_ADDRESS6=${cfg.hostAddress6} + ''} ${optionalString (cfg.localAddress != null) '' LOCAL_ADDRESS=${cfg.localAddress} ''} + ${optionalString (cfg.localAddress6 != null) '' + LOCAL_ADDRESS6=${cfg.localAddress6} + ''} ''} INTERFACES="${toString cfg.interfaces}" ${optionalString cfg.autoStart '' -- cgit 1.4.1 From 3c819f28f57ee1485f15faf917f2fc6a861a6883 Mon Sep 17 00:00:00 2001 From: Arnold Krille Date: Sat, 2 Apr 2016 17:03:30 +0200 Subject: containers: Make declarative containers real systemd services Without the templating (which is still present for imperative containers), it will be possible to set individual dependencies. Like depending on the network only if the hostbridge or hardware interfaces are used. Ported from #3021 --- nixos/modules/virtualisation/containers.nix | 307 +++++++++++++--------------- 1 file changed, 147 insertions(+), 160 deletions(-) (limited to 'nixos/modules') diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 4c20ee27de2c..fca21a8610be 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -278,167 +278,180 @@ in }; - config = mkIf (config.boot.enableContainers) { + config = mkIf (config.boot.enableContainers) (let - systemd.services."container@" = - { description = "Container '%i'"; + unit = { + description = "Container '%i'"; - unitConfig.RequiresMountsFor = [ "/var/lib/containers/%i" ]; + unitConfig.RequiresMountsFor = [ "/var/lib/containers/%i" ]; - path = [ pkgs.iproute ]; + path = [ pkgs.iproute ]; - environment.INSTANCE = "%i"; - environment.root = "/var/lib/containers/%i"; + environment.INSTANCE = "%i"; + environment.root = "/var/lib/containers/%i"; - preStart = - '' - # Clean up existing machined registration and interfaces. - machinectl terminate "$INSTANCE" 2> /dev/null || true + preStart = + '' + # Clean up existing machined registration and interfaces. + machinectl terminate "$INSTANCE" 2> /dev/null || true - if [ "$PRIVATE_NETWORK" = 1 ]; then - ip link del dev "ve-$INSTANCE" 2> /dev/null || true - ip link del dev "vb-$INSTANCE" 2> /dev/null || true - fi - ''; + if [ "$PRIVATE_NETWORK" = 1 ]; then + ip link del dev "ve-$INSTANCE" 2> /dev/null || true + ip link del dev "vb-$INSTANCE" 2> /dev/null || true + fi + ''; - script = - '' - mkdir -p -m 0755 "$root/etc" "$root/var/lib" - mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/containers - if ! [ -e "$root/etc/os-release" ]; then - touch "$root/etc/os-release" + script = + '' + mkdir -p -m 0755 "$root/etc" "$root/var/lib" + mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/containers + if ! [ -e "$root/etc/os-release" ]; then + touch "$root/etc/os-release" + fi + + mkdir -p -m 0755 \ + "/nix/var/nix/profiles/per-container/$INSTANCE" \ + "/nix/var/nix/gcroots/per-container/$INSTANCE" + + cp --remove-destination /etc/resolv.conf "$root/etc/resolv.conf" + + if [ "$PRIVATE_NETWORK" = 1 ]; then + extraFlags+=" --network-veth" + if [ -n "$HOST_BRIDGE" ]; then + extraFlags+=" --network-bridge=$HOST_BRIDGE" fi + fi - mkdir -p -m 0755 \ - "/nix/var/nix/profiles/per-container/$INSTANCE" \ - "/nix/var/nix/gcroots/per-container/$INSTANCE" + for iface in $INTERFACES; do + extraFlags+=" --network-interface=$iface" + done - cp --remove-destination /etc/resolv.conf "$root/etc/resolv.conf" + for iface in $MACVLANS; do + extraFlags+=" --network-macvlan=$iface" + done - if [ "$PRIVATE_NETWORK" = 1 ]; then - extraFlags+=" --network-veth" - if [ -n "$HOST_BRIDGE" ]; then - extraFlags+=" --network-bridge=$HOST_BRIDGE" - fi + # If the host is 64-bit and the container is 32-bit, add a + # --personality flag. + ${optionalString (config.nixpkgs.system == "x86_64-linux") '' + if [ "$(< ''${SYSTEM_PATH:-/nix/var/nix/profiles/per-container/$INSTANCE/system}/system)" = i686-linux ]; then + extraFlags+=" --personality=x86" fi + ''} + + + + # Run systemd-nspawn without startup notification (we'll + # wait for the container systemd to signal readiness). + EXIT_ON_REBOOT=1 NOTIFY_SOCKET= \ + exec ${config.systemd.package}/bin/systemd-nspawn \ + --keep-unit \ + -M "$INSTANCE" -D "$root" $extraFlags \ + $EXTRA_NSPAWN_FLAGS \ + --bind-ro=/nix/store \ + --bind-ro=/nix/var/nix/db \ + --bind-ro=/nix/var/nix/daemon-socket \ + --bind=/run/systemd/notify:/var/lib/private/host-notify \ + --bind="/nix/var/nix/profiles/per-container/$INSTANCE:/nix/var/nix/profiles" \ + --bind="/nix/var/nix/gcroots/per-container/$INSTANCE:/nix/var/nix/gcroots" \ + --setenv PRIVATE_NETWORK="$PRIVATE_NETWORK" \ + --setenv HOST_BRIDGE="$HOST_BRIDGE" \ + --setenv HOST_ADDRESS="$HOST_ADDRESS" \ + --setenv LOCAL_ADDRESS="$LOCAL_ADDRESS" \ + --setenv HOST_ADDRESS6="$HOST_ADDRESS6" \ + --setenv LOCAL_ADDRESS6="$LOCAL_ADDRESS6" \ + --setenv PATH="$PATH" \ + ${containerInit} "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/init" + ''; - for iface in $INTERFACES; do - extraFlags+=" --network-interface=$iface" - done - - for iface in $MACVLANS; do - extraFlags+=" --network-macvlan=$iface" - done - - # If the host is 64-bit and the container is 32-bit, add a - # --personality flag. - ${optionalString (config.nixpkgs.system == "x86_64-linux") '' - if [ "$(< ''${SYSTEM_PATH:-/nix/var/nix/profiles/per-container/$INSTANCE/system}/system)" = i686-linux ]; then - extraFlags+=" --personality=x86" + postStart = + '' + if [ "$PRIVATE_NETWORK" = 1 ]; then + if [ -z "$HOST_BRIDGE" ]; then + ifaceHost=ve-$INSTANCE + ip link set dev $ifaceHost up + if [ -n "$HOST_ADDRESS" ]; then + ip addr add $HOST_ADDRESS dev $ifaceHost fi - ''} - - - - # Run systemd-nspawn without startup notification (we'll - # wait for the container systemd to signal readiness). - EXIT_ON_REBOOT=1 NOTIFY_SOCKET= \ - exec ${config.systemd.package}/bin/systemd-nspawn \ - --keep-unit \ - -M "$INSTANCE" -D "$root" $extraFlags \ - $EXTRA_NSPAWN_FLAGS \ - --bind-ro=/nix/store \ - --bind-ro=/nix/var/nix/db \ - --bind-ro=/nix/var/nix/daemon-socket \ - --bind=/run/systemd/notify:/var/lib/private/host-notify \ - --bind="/nix/var/nix/profiles/per-container/$INSTANCE:/nix/var/nix/profiles" \ - --bind="/nix/var/nix/gcroots/per-container/$INSTANCE:/nix/var/nix/gcroots" \ - --setenv PRIVATE_NETWORK="$PRIVATE_NETWORK" \ - --setenv HOST_BRIDGE="$HOST_BRIDGE" \ - --setenv HOST_ADDRESS="$HOST_ADDRESS" \ - --setenv LOCAL_ADDRESS="$LOCAL_ADDRESS" \ - --setenv HOST_ADDRESS6="$HOST_ADDRESS6" \ - --setenv LOCAL_ADDRESS6="$LOCAL_ADDRESS6" \ - --setenv PATH="$PATH" \ - ${containerInit} "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/init" - ''; - - postStart = - '' - if [ "$PRIVATE_NETWORK" = 1 ]; then - if [ -z "$HOST_BRIDGE" ]; then - ifaceHost=ve-$INSTANCE - ip link set dev $ifaceHost up - if [ -n "$HOST_ADDRESS" ]; then - ip addr add $HOST_ADDRESS dev $ifaceHost - fi - if [ -n "$HOST_ADDRESS6" ]; then - ip -6 addr add $HOST_ADDRESS6 dev $ifaceHost - fi - if [ -n "$LOCAL_ADDRESS" ]; then - ip route add $LOCAL_ADDRESS dev $ifaceHost - fi - if [ -n "$LOCAL_ADDRESS6" ]; then - ip -6 route add $LOCAL_ADDRESS6 dev $ifaceHost - fi + if [ -n "$HOST_ADDRESS6" ]; then + ip -6 addr add $HOST_ADDRESS6 dev $ifaceHost + fi + if [ -n "$LOCAL_ADDRESS" ]; then + ip route add $LOCAL_ADDRESS dev $ifaceHost + fi + if [ -n "$LOCAL_ADDRESS6" ]; then + ip -6 route add $LOCAL_ADDRESS6 dev $ifaceHost fi fi + fi - # Get the leader PID so that we can signal it in - # preStop. We can't use machinectl there because D-Bus - # might be shutting down. FIXME: in systemd 219 we can - # just signal systemd-nspawn to do a clean shutdown. - machinectl show "$INSTANCE" | sed 's/Leader=\(.*\)/\1/;t;d' > "/run/containers/$INSTANCE.pid" - ''; - - preStop = - '' - pid="$(cat /run/containers/$INSTANCE.pid)" - if [ -n "$pid" ]; then - kill -RTMIN+4 "$pid" - fi - rm -f "/run/containers/$INSTANCE.pid" - ''; + # Get the leader PID so that we can signal it in + # preStop. We can't use machinectl there because D-Bus + # might be shutting down. FIXME: in systemd 219 we can + # just signal systemd-nspawn to do a clean shutdown. + machinectl show "$INSTANCE" | sed 's/Leader=\(.*\)/\1/;t;d' > "/run/containers/$INSTANCE.pid" + ''; - restartIfChanged = false; - #reloadIfChanged = true; # FIXME + preStop = + '' + pid="$(cat /run/containers/$INSTANCE.pid)" + if [ -n "$pid" ]; then + kill -RTMIN+4 "$pid" + fi + rm -f "/run/containers/$INSTANCE.pid" + ''; - wants = [ "netwprk.target" ]; - after = [ "network.target" ]; + restartIfChanged = false; - serviceConfig = { - ExecReload = pkgs.writeScript "reload-container" - '' - #! ${pkgs.stdenv.shell} -e - ${nixos-container}/bin/nixos-container run "$INSTANCE" -- \ - bash --login -c "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/bin/switch-to-configuration test" - ''; + serviceConfig = { + ExecReload = pkgs.writeScript "reload-container" + '' + #! ${pkgs.stdenv.shell} -e + ${nixos-container}/bin/nixos-container run "$INSTANCE" -- \ + bash --login -c "''${SYSTEM_PATH:-/nix/var/nix/profiles/system}/bin/switch-to-configuration test" + ''; - SyslogIdentifier = "container %i"; + SyslogIdentifier = "container %i"; - EnvironmentFile = "-/etc/containers/%i.conf"; + EnvironmentFile = "-/etc/containers/%i.conf"; - Type = "notify"; + Type = "notify"; - NotifyAccess = "all"; + NotifyAccess = "all"; - # Note that on reboot, systemd-nspawn returns 133, so this - # unit will be restarted. On poweroff, it returns 0, so the - # unit won't be restarted. - RestartForceExitStatus = "133"; - SuccessExitStatus = "133"; + # Note that on reboot, systemd-nspawn returns 133, so this + # unit will be restarted. On poweroff, it returns 0, so the + # unit won't be restarted. + RestartForceExitStatus = "133"; + SuccessExitStatus = "133"; - Restart = "on-failure"; + Restart = "on-failure"; - # Hack: we don't want to kill systemd-nspawn, since we call - # "machinectl poweroff" in preStop to shut down the - # container cleanly. But systemd requires sending a signal - # (at least if we want remaining processes to be killed - # after the timeout). So send an ignored signal. - KillMode = "mixed"; - KillSignal = "WINCH"; - }; + # Hack: we don't want to kill systemd-nspawn, since we call + # "machinectl poweroff" in preStop to shut down the + # container cleanly. But systemd requires sending a signal + # (at least if we want remaining processes to be killed + # after the timeout). So send an ignored signal. + KillMode = "mixed"; + KillSignal = "WINCH"; }; + }; + in { + systemd.services = listToAttrs (filter (x: x.value != null) ( + # The generic container template used by imperative containers + [{ name = "container@"; value = unit; }] + # declarative containers + ++ (mapAttrsToList (name: cfg: nameValuePair "container@${name}" ( + if cfg.autoStart then + unit // { + wantedBy = [ "multi-user.target" ]; + wants = [ "network.target" ]; + after = [ "network.target" ]; + restartTriggers = [ cfg.path ]; + reloadIfChanged = true; + } + else null + )) config.containers) + )); # Generate a configuration file in /etc/containers for each # container so that container@.target can get the container @@ -482,31 +495,5 @@ in networking.dhcpcd.denyInterfaces = [ "ve-*" ]; environment.systemPackages = [ nixos-container ]; - - # Start containers at boot time. - systemd.services.all-containers = - { description = "All Containers"; - - wantedBy = [ "multi-user.target" ]; - - unitConfig.ConditionDirectoryNotEmpty = "/etc/containers"; - - serviceConfig.Type = "oneshot"; - - script = - '' - res=0 - shopt -s nullglob - for i in /etc/containers/*.conf; do - AUTO_START= - source "$i" - if [ "$AUTO_START" = 1 ]; then - systemctl start "container@$(basename "$i" .conf).service" || res=1 - fi - done - exit $res - ''; # */ - }; - - }; + }); } -- cgit 1.4.1