summary refs log tree commit diff
diff options
context:
space:
mode:
-rw-r--r--doc/languages-frameworks/python.section.md12
-rw-r--r--nixos/doc/manual/release-notes/rl-1809.xml6
-rw-r--r--nixos/lib/qemu-flags.nix8
-rw-r--r--nixos/modules/module-list.nix2
-rw-r--r--nixos/modules/services/development/jupyter/default.nix184
-rw-r--r--nixos/modules/services/development/jupyter/kernel-options.nix60
-rw-r--r--nixos/modules/services/search/elasticsearch-curator.nix93
-rw-r--r--nixos/tests/ec2.nix4
-rw-r--r--nixos/tests/elk.nix32
-rw-r--r--nixos/tests/matrix-synapse.nix17
-rw-r--r--nixos/tests/networking.nix12
-rw-r--r--pkgs/applications/audio/split2flac/default.nix62
-rw-r--r--pkgs/applications/editors/jupyter/default.nix18
-rw-r--r--pkgs/applications/editors/jupyter/kernel.nix74
-rw-r--r--pkgs/applications/misc/subsurface/default.nix6
-rw-r--r--pkgs/applications/networking/firehol/default.nix4
-rw-r--r--pkgs/applications/networking/instant-messengers/skypeforlinux/default.nix4
-rw-r--r--pkgs/applications/networking/irc/konversation/default.nix11
-rw-r--r--pkgs/applications/virtualization/qemu/default.nix7
-rw-r--r--pkgs/desktops/gnome-3/default.nix2
-rw-r--r--pkgs/desktops/gnome-3/games/quadrapassel/default.nix30
-rw-r--r--pkgs/development/libraries/cmrt/default.nix24
-rw-r--r--pkgs/development/libraries/libmanette/default.nix36
-rw-r--r--pkgs/development/libraries/mac/default.nix36
-rw-r--r--pkgs/development/libraries/qt-5/modules/qtwebengine.nix2
-rw-r--r--pkgs/development/libraries/vaapi-intel-hybrid/default.nix47
-rw-r--r--pkgs/development/libraries/vaapi-intel/default.nix16
-rw-r--r--pkgs/development/python-modules/elasticsearch-curator/default.nix10
-rw-r--r--pkgs/development/python-modules/requests-aws4auth/default.nix28
-rw-r--r--pkgs/development/tools/misc/tokei/default.nix6
-rw-r--r--pkgs/games/20kly/default.nix40
-rw-r--r--pkgs/os-specific/linux/firmware/fwupdate/default.nix6
-rw-r--r--pkgs/os-specific/linux/firmware/fwupdate/fix-paths.patch116
-rw-r--r--pkgs/os-specific/linux/kernel/common-config.nix8
-rw-r--r--pkgs/top-level/all-packages.nix16
-rw-r--r--pkgs/top-level/python-packages.nix2
36 files changed, 856 insertions, 185 deletions
diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md
index 93eb5af0f2cc..d825823d01a2 100644
--- a/doc/languages-frameworks/python.section.md
+++ b/doc/languages-frameworks/python.section.md
@@ -645,9 +645,15 @@ in python.withPackages(ps: [ps.blaze])).env
 
 #### `buildPythonApplication` function
 
-The `buildPythonApplication` function is practically the same as `buildPythonPackage`.
-The difference is that `buildPythonPackage` by default prefixes the names of the packages with the version of the interpreter.
-Because this is irrelevant for applications, the prefix is omitted.
+The `buildPythonApplication` function is practically the same as
+`buildPythonPackage`. The main purpose of this function is to build a Python
+package where one is interested only in the executables, and not importable
+modules. For that reason, when adding this package to a `python.buildEnv`, the
+modules won't be made available.
+
+Another difference is that `buildPythonPackage` by default prefixes the names of
+the packages with the version of the interpreter. Because this is irrelevant for
+applications, the prefix is omitted.
 
 #### `toPythonApplication` function
 
diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml
index 8ee2a5f16238..d190394b9887 100644
--- a/nixos/doc/manual/release-notes/rl-1809.xml
+++ b/nixos/doc/manual/release-notes/rl-1809.xml
@@ -111,6 +111,12 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull'
       <link xlink:href="https://github.com/strongswan/strongswan/blob/master/README_LEGACY.md">stroke configuration interface</link>.
     </para>
    </listitem>
+   <listitem>
+    <para>
+      The new <varname>services.elasticsearch-curator</varname> service
+      periodically curates or manages, your Elasticsearch indices and snapshots.
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 
diff --git a/nixos/lib/qemu-flags.nix b/nixos/lib/qemu-flags.nix
index 6f61c64a832e..f115ca5ac000 100644
--- a/nixos/lib/qemu-flags.nix
+++ b/nixos/lib/qemu-flags.nix
@@ -1,11 +1,15 @@
 # QEMU flags shared between various Nix expressions.
 { pkgs }:
 
+let
+  zeroPad = n: if n < 10 then "0${toString n}" else toString n;
+in
+
 {
 
   qemuNICFlags = nic: net: machine:
-    [ "-net nic,vlan=${toString nic},macaddr=52:54:00:12:${toString net}:${toString machine},model=virtio"
-      "-net vde,vlan=${toString nic},sock=$QEMU_VDE_SOCKET_${toString net}"
+    [ "-device virtio-net-pci,netdev=vlan${toString nic},mac=52:54:00:12:${zeroPad net}:${zeroPad machine}"
+      "-netdev vde,id=vlan${toString nic},sock=$QEMU_VDE_SOCKET_${toString net}"
     ];
 
   qemuSerialDevice = if pkgs.stdenv.isi686 || pkgs.stdenv.isx86_64 then "ttyS0"
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 2846afea8fbc..12944857af4e 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -250,6 +250,7 @@
   ./services/desktops/zeitgeist.nix
   ./services/development/bloop.nix
   ./services/development/hoogle.nix
+  ./services/development/jupyter/default.nix
   ./services/editors/emacs.nix
   ./services/editors/infinoted.nix
   ./services/games/factorio.nix
@@ -623,6 +624,7 @@
   ./services/scheduling/fcron.nix
   ./services/scheduling/marathon.nix
   ./services/search/elasticsearch.nix
+  ./services/search/elasticsearch-curator.nix
   ./services/search/hound.nix
   ./services/search/kibana.nix
   ./services/search/solr.nix
diff --git a/nixos/modules/services/development/jupyter/default.nix b/nixos/modules/services/development/jupyter/default.nix
new file mode 100644
index 000000000000..9fcc00431865
--- /dev/null
+++ b/nixos/modules/services/development/jupyter/default.nix
@@ -0,0 +1,184 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.services.jupyter;
+
+  # NOTE: We don't use top-level jupyter because we don't
+  # want to pass in JUPYTER_PATH but use .environment instead,
+  # saving a rebuild.
+  package = pkgs.python3.pkgs.notebook;
+
+  kernels = (pkgs.jupyter-kernel.create  {
+    definitions = if cfg.kernels != null
+      then cfg.kernels
+      else  pkgs.jupyter-kernel.default;
+  });
+
+  notebookConfig = pkgs.writeText "jupyter_config.py" ''
+    ${cfg.notebookConfig}
+
+    c.NotebookApp.password = ${cfg.password}
+  '';
+
+in {
+  meta.maintainers = with maintainers; [ aborsu ];
+
+  options.services.jupyter = {
+    enable = mkEnableOption "Jupyter development server";
+
+    ip = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = ''
+        IP address Jupyter will be listening on.
+      '';
+    };
+
+    port = mkOption {
+      type = types.int;
+      default = 8888;
+      description = ''
+        Port number Jupyter will be listening on.
+      '';
+    };
+
+    notebookDir = mkOption {
+      type = types.str;
+      default = "~/";
+      description = ''
+        Root directory for notebooks.
+      '';
+    };
+
+    user = mkOption {
+      type = types.str;
+      default = "jupyter";
+      description = ''
+        Name of the user used to run the jupyter service.
+        For security reason, jupyter should really not be run as root.
+        If not set (jupyter), the service will create a jupyter user with appropriate settings.
+      '';
+      example = "aborsu";
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = "jupyter";
+      description = ''
+        Name of the group used to run the jupyter service.
+        Use this if you want to create a group of users that are able to view the notebook directory's content.
+      '';
+      example = "users";
+    };
+
+    password = mkOption {
+      type = types.str;
+      description = ''
+        Password to use with notebook.
+        Can be generated using:
+          In [1]: from notebook.auth import passwd
+          In [2]: passwd('test')
+          Out[2]: 'sha1:1b961dc713fb:88483270a63e57d18d43cf337e629539de1436ba'
+          NOTE: you need to keep the single quote inside the nix string.
+        Or you can use a python oneliner:
+          "open('/path/secret_file', 'r', encoding='utf8').read().strip()"
+        It will be interpreted at the end of the notebookConfig.
+      '';
+      example = [
+        "'sha1:1b961dc713fb:88483270a63e57d18d43cf337e629539de1436ba'"
+        "open('/path/secret_file', 'r', encoding='utf8').read().strip()"
+      ];
+    };
+
+    notebookConfig = mkOption {
+      type = types.lines;
+      default = "";
+      description = ''
+        Raw jupyter config.
+      '';
+    };
+
+    kernels = mkOption {
+      type = types.nullOr (types.attrsOf(types.submodule (import ./kernel-options.nix {
+        inherit lib;
+      })));
+
+      default = null;
+      example = literalExample ''
+        {
+          python3 = let
+            env = (pkgs.python3.withPackages (pythonPackages: with pythonPackages; [
+                    ipykernel
+                    pandas
+                    scikitlearn
+                  ]));
+          in {
+            displayName = "Python 3 for machine learning";
+            argv = [
+              "$ {env.interpreter}"
+              "-m"
+              "ipykernel_launcher"
+              "-f"
+              "{connection_file}"
+            ];
+            language = "python";
+            logo32 = "$ {env.sitePackages}/ipykernel/resources/logo-32x32.png";
+            logo64 = "$ {env.sitePackages}/ipykernel/resources/logo-64x64.png";
+          };
+        }
+      '';
+      description = "Declarative kernel config
+
+      Kernels can be declared in any language that supports and has the required
+      dependencies to communicate with a jupyter server.
+      In python's case, it means that ipykernel package must always be included in
+      the list of packages of the targeted environment.
+      ";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.enable  {
+      systemd.services.jupyter = {
+        description = "Jupyter development server";
+
+        wantedBy = [ "multi-user.target" ];
+
+        # TODO: Patch notebook so we can explicitly pass in a shell
+        path = [ pkgs.bash ]; # needed for sh in cell magic to work
+
+        environment = {
+          JUPYTER_PATH = toString kernels;
+        };
+
+        serviceConfig = {
+          Restart = "always";
+          ExecStart = ''${package}/bin/jupyter-notebook \
+            --no-browser \
+            --ip=${cfg.ip} \
+            --port=${toString cfg.port} --port-retries 0 \
+            --notebook-dir=${cfg.notebookDir} \
+            --NotebookApp.config_file=${notebookConfig}
+          '';
+          User = cfg.user;
+          Group = cfg.group;
+          WorkingDirectory = "~";
+        };
+      };
+    })
+    (mkIf (cfg.enable && (cfg.group == "jupyter")) {
+      users.groups.jupyter = {};
+    })
+    (mkIf (cfg.enable && (cfg.user == "jupyter")) {
+      users.extraUsers.jupyter = {
+        extraGroups = [ cfg.group ];
+        home = "/var/lib/jupyter";
+        createHome = true;
+        useDefaultShell = true; # needed so that the user can start a terminal.
+      };
+    })
+  ];
+}
diff --git a/nixos/modules/services/development/jupyter/kernel-options.nix b/nixos/modules/services/development/jupyter/kernel-options.nix
new file mode 100644
index 000000000000..03547637449a
--- /dev/null
+++ b/nixos/modules/services/development/jupyter/kernel-options.nix
@@ -0,0 +1,60 @@
+# Options that can be used for creating a jupyter kernel.
+{lib }:
+
+with lib;
+
+{
+  options = {
+
+    displayName = mkOption {
+      type = types.str;
+      default = "";
+      example = [
+        "Python 3"
+        "Python 3 for Data Science"
+      ];
+      description = ''
+        Name that will be shown to the user.
+      '';
+    };
+
+    argv = mkOption {
+      type = types.listOf types.str;
+      example = [
+        "{customEnv.interpreter}"
+        "-m"
+        "ipykernel_launcher"
+        "-f"
+        "{connection_file}"
+      ];
+      description = ''
+        Command and arguments to start the kernel.
+      '';
+    };
+
+    language = mkOption {
+      type = types.str;
+      example = "python";
+      description = ''
+        Language of the environment. Typically the name of the binary.
+      '';
+    };
+
+    logo32 = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      example = "{env.sitePackages}/ipykernel/resources/logo-32x32.png";
+      description = ''
+        Path to 32x32 logo png.
+      '';
+    };
+    logo64 = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      example = "{env.sitePackages}/ipykernel/resources/logo-64x64.png";
+      description = ''
+        Path to 64x64 logo png.
+      '';
+    };
+  };
+}
diff --git a/nixos/modules/services/search/elasticsearch-curator.nix b/nixos/modules/services/search/elasticsearch-curator.nix
new file mode 100644
index 000000000000..43785c392fee
--- /dev/null
+++ b/nixos/modules/services/search/elasticsearch-curator.nix
@@ -0,0 +1,93 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+    cfg = config.services.elasticsearch-curator;
+    curatorConfig = pkgs.writeTextFile {
+      name = "config.yaml";
+      text = ''
+        ---
+        # Remember, leave a key empty if there is no value.  None will be a string,
+        # not a Python "NoneType"
+        client:
+          hosts: ${builtins.toJSON cfg.hosts}
+          port: ${toString cfg.port}
+          url_prefix:
+          use_ssl: False
+          certificate:
+          client_cert:
+          client_key:
+          ssl_no_validate: False
+          http_auth:
+          timeout: 30
+          master_only: False
+        logging:
+          loglevel: INFO
+          logfile:
+          logformat: default
+          blacklist: ['elasticsearch', 'urllib3']
+        '';
+    };
+    curatorAction = pkgs.writeTextFile {
+      name = "action.yaml";
+      text = cfg.actionYAML;
+    };
+in {
+
+  options.services.elasticsearch-curator = {
+
+    enable = mkEnableOption "elasticsearch curator";
+    interval = mkOption {
+      description = "The frequency to run curator, a systemd.time such as 'hourly'";
+      default = "hourly";
+      type = types.str;
+    };
+    hosts = mkOption {
+      description = "a list of elasticsearch hosts to connect to";
+      type = types.listOf types.str;
+      default = ["localhost"];
+    };
+    port = mkOption {
+      description = "the port that elasticsearch is listening on";
+      type = types.int;
+      default = 9200;
+    };
+    actionYAML = mkOption {
+      description = "curator action.yaml file contents, alternatively use curator-cli which takes a simple action command";
+      example = ''
+        ---
+        actions:
+          1:
+            action: delete_indices
+            description: >-
+              Delete indices older than 45 days (based on index name), for logstash-
+              prefixed indices. Ignore the error if the filter does not result in an
+              actionable list of indices (ignore_empty_list) and exit cleanly.
+            options:
+              ignore_empty_list: True
+              disable_action: False
+            filters:
+            - filtertype: pattern
+              kind: prefix
+              value: logstash-
+            - filtertype: age
+              source: name
+              direction: older
+              timestring: '%Y.%m.%d'
+              unit: days
+              unit_count: 45
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    systemd.services.elasticsearch-curator = {
+      startAt = cfg.interval;
+      serviceConfig = {
+        ExecStart = ''${pkgs.python36Packages.elasticsearch-curator}/bin/curator --config ${curatorConfig} ${curatorAction}'';
+      };
+    };
+  };
+}
diff --git a/nixos/tests/ec2.nix b/nixos/tests/ec2.nix
index f585fa2ec237..8271747ccc63 100644
--- a/nixos/tests/ec2.nix
+++ b/nixos/tests/ec2.nix
@@ -68,7 +68,9 @@ let
           # again when it deletes link-local addresses.) Ideally we'd
           # turn off the DHCP server, but qemu does not have an option
           # to do that.
-          my $startCommand = "qemu-kvm -m 768 -net nic,vlan=0,model=virtio -net 'user,vlan=0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd ${metaData}'";
+          my $startCommand = "qemu-kvm -m 768";
+          $startCommand .= " -device virtio-net-pci,netdev=vlan0";
+          $startCommand .= " -netdev 'user,id=vlan0,net=169.0.0.0/8,guestfwd=tcp:169.254.169.254:80-cmd:${pkgs.micro-httpd}/bin/micro_httpd ${metaData}'";
           $startCommand .= " -drive file=$diskImage,if=virtio,werror=report";
           $startCommand .= " \$QEMU_OPTS";
 
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index 4c5c441ca265..15be72b80bba 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -63,6 +63,33 @@ let
                 package = elk.kibana;
                 elasticsearch.url = esUrl;
               };
+
+              elasticsearch-curator = {
+                enable = true;
+                actionYAML = ''
+                ---
+                actions:
+                  1:
+                    action: delete_indices
+                    description: >-
+                      Delete indices older than 1 second (based on index name), for logstash-
+                      prefixed indices. Ignore the error if the filter does not result in an
+                      actionable list of indices (ignore_empty_list) and exit cleanly.
+                    options:
+                      ignore_empty_list: True
+                      disable_action: False
+                    filters:
+                    - filtertype: pattern
+                      kind: prefix
+                      value: logstash-
+                    - filtertype: age
+                      source: name
+                      direction: older
+                      timestring: '%Y.%m.%d'
+                      unit: seconds
+                      unit_count: 1
+                '';
+              };
             };
           };
       };
@@ -91,6 +118,11 @@ let
       # See if logstash messages arive in elasticsearch.
       $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
       $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
+
+      # Test elasticsearch-curator.
+      $one->systemctl("stop logstash");
+      $one->systemctl("start elasticsearch-curator");
+      $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
     '';
   };
 in mapAttrs mkElkTest {
diff --git a/nixos/tests/matrix-synapse.nix b/nixos/tests/matrix-synapse.nix
index 113fb622588b..8504a7c0d057 100644
--- a/nixos/tests/matrix-synapse.nix
+++ b/nixos/tests/matrix-synapse.nix
@@ -6,12 +6,13 @@ import ./make-test.nix ({ pkgs, ... } : {
   };
 
   nodes = {
-    server_postgres = args: {
+    # Since 0.33.0, matrix-synapse doesn't allow underscores in server names
+    serverpostgres = args: {
       services.matrix-synapse.enable = true;
       services.matrix-synapse.database_type = "psycopg2";
     };
 
-    server_sqlite = args: {
+    serversqlite = args: {
       services.matrix-synapse.enable = true;
       services.matrix-synapse.database_type = "sqlite3";
     };
@@ -19,12 +20,12 @@ import ./make-test.nix ({ pkgs, ... } : {
 
   testScript = ''
     startAll;
-    $server_postgres->waitForUnit("matrix-synapse.service");
-    $server_postgres->waitUntilSucceeds("curl -Lk https://localhost:8448/");
-    $server_postgres->requireActiveUnit("postgresql.service");
-    $server_sqlite->waitForUnit("matrix-synapse.service");
-    $server_sqlite->waitUntilSucceeds("curl -Lk https://localhost:8448/");
-    $server_sqlite->mustSucceed("[ -e /var/lib/matrix-synapse/homeserver.db ]");
+    $serverpostgres->waitForUnit("matrix-synapse.service");
+    $serverpostgres->waitUntilSucceeds("curl -Lk https://localhost:8448/");
+    $serverpostgres->requireActiveUnit("postgresql.service");
+    $serversqlite->waitForUnit("matrix-synapse.service");
+    $serversqlite->waitUntilSucceeds("curl -Lk https://localhost:8448/");
+    $serversqlite->mustSucceed("[ -e /var/lib/matrix-synapse/homeserver.db ]");
   '';
 
 })
diff --git a/nixos/tests/networking.nix b/nixos/tests/networking.nix
index cd5d537a3be7..ff088ad2621b 100644
--- a/nixos/tests/networking.nix
+++ b/nixos/tests/networking.nix
@@ -448,8 +448,8 @@ let
 
       testScript = ''
         my $targetList = <<'END';
-        tap0: tap UNKNOWN_FLAGS:800 user 0
-        tun0: tun UNKNOWN_FLAGS:800 user 0
+        tap0: tap persist user 0
+        tun0: tun persist user 0
         END
 
         # Wait for networking to come up
@@ -552,15 +552,15 @@ let
 
       testScript = ''
         my $targetIPv4Table = <<'END';
-        10.0.0.0/16 scope link mtu 1500 
+        10.0.0.0/16 proto static scope link mtu 1500 
         192.168.1.0/24 proto kernel scope link src 192.168.1.2 
-        192.168.2.0/24 via 192.168.1.1 
+        192.168.2.0/24 via 192.168.1.1 proto static 
         END
 
         my $targetIPv6Table = <<'END';
         2001:1470:fffd:2097::/64 proto kernel metric 256 pref medium
-        2001:1470:fffd:2098::/64 via fdfd:b3f0::1 metric 1024 pref medium
-        fdfd:b3f0::/48 metric 1024 pref medium
+        2001:1470:fffd:2098::/64 via fdfd:b3f0::1 proto static metric 1024 pref medium
+        fdfd:b3f0::/48 proto static metric 1024 pref medium
         END
 
         $machine->start;
diff --git a/pkgs/applications/audio/split2flac/default.nix b/pkgs/applications/audio/split2flac/default.nix
new file mode 100644
index 000000000000..372e507ed0e9
--- /dev/null
+++ b/pkgs/applications/audio/split2flac/default.nix
@@ -0,0 +1,62 @@
+{ stdenv, fetchFromGitHub, makeWrapper
+, shntool, cuetools
+, flac, faac, mp4v2, wavpack, mac
+, imagemagick, libiconv, enca, lame, pythonPackages, vorbis-tools
+, aacgain, mp3gain, vorbisgain
+}:
+
+let
+  wrapSplit2flac =  format: ''
+    makeWrapper $out/bin/.split2flac-wrapped $out/bin/split2${format} \
+      --set SPLIT2FLAC_FORMAT ${format} \
+      --prefix PATH : ${stdenv.lib.makeBinPath [
+        shntool cuetools
+        flac faac mp4v2 wavpack mac
+        imagemagick libiconv enca lame pythonPackages.mutagen vorbis-tools
+        aacgain mp3gain vorbisgain
+      ]}
+  '';
+
+in stdenv.mkDerivation rec {
+  name = "split2flac-${version}";
+  version = "122";
+
+  src = fetchFromGitHub {
+    owner = "ftrvxmtrx";
+    repo = "split2flac";
+    rev = version;
+    sha256 = "1a71amamip25hhqx7wwzfcl3d5snry9xsiha0kw73iq2m83r2k63";
+  };
+
+  dontBuild = true;
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  patchPhase = ''
+    substituteInPlace split2flac \
+      --replace 'FORMAT="''${0##*split2}"' 'FORMAT=''${SPLIT2FLAC_FORMAT:-flac}'
+  '';
+
+  installPhase = ''
+    mkdir -p $out/share/bash-completion/completions
+    cp split2flac-bash-completion.sh \
+      $out/share/bash-completion/completions/split2flac-bash-completion.sh
+
+    mkdir -p $out/bin
+    cp split2flac $out/bin/.split2flac-wrapped
+
+    ${wrapSplit2flac "flac"}
+    ${wrapSplit2flac "mp3"}
+    ${wrapSplit2flac "ogg"}
+    ${wrapSplit2flac "m4a"}
+    ${wrapSplit2flac "wav"}
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Split flac/ape/wv/wav + cue sheet into separate tracks";
+    homepage = https://github.com/ftrvxmtrx/split2flac;
+    license = licenses.mit;
+    platforms = platforms.all;
+    maintainers = with maintainers; [ jfrankenau ];
+  };
+}
diff --git a/pkgs/applications/editors/jupyter/default.nix b/pkgs/applications/editors/jupyter/default.nix
new file mode 100644
index 000000000000..2bca120c1deb
--- /dev/null
+++ b/pkgs/applications/editors/jupyter/default.nix
@@ -0,0 +1,18 @@
+# Jupyter notebook with the given kernel definitions
+
+{ python3
+, jupyter-kernel
+, definitions ? jupyter-kernel.default
+}:
+
+let
+
+  jupyterPath = (jupyter-kernel.create { inherit definitions; });
+
+in
+
+with python3.pkgs; toPythonModule (
+  notebook.overridePythonAttrs(oldAttrs: {
+    makeWrapperArgs = ["--set JUPYTER_PATH ${jupyterPath}"];
+  })
+)
diff --git a/pkgs/applications/editors/jupyter/kernel.nix b/pkgs/applications/editors/jupyter/kernel.nix
new file mode 100644
index 000000000000..cdb6b6212bd3
--- /dev/null
+++ b/pkgs/applications/editors/jupyter/kernel.nix
@@ -0,0 +1,74 @@
+{ lib, stdenv, python3}:
+
+let
+
+  default = {
+    python3 = let
+      env = (python3.withPackages (ps: with ps; [ ipykernel ]));
+    in {
+      displayName = "Python 3";
+      argv = [
+        "${env.interpreter}"
+        "-m"
+        "ipykernel_launcher"
+        "-f"
+        "{connection_file}"
+      ];
+      language = "python";
+      logo32 = "${env.sitePackages}/ipykernel/resources/logo-32x32.png";
+      logo64 = "${env.sitePackages}/ipykernel/resources/logo-64x64.png";
+    };
+  };
+
+in
+{
+  inherit default;
+
+  # Definitions is an attribute set.
+
+  create = { definitions ?  default }: with lib; stdenv.mkDerivation rec {
+
+    name = "jupyter-kernels";
+
+    src = "/dev/null";
+
+    unpackCmd = "mkdir jupyter_kernels";
+
+    installPhase =  ''
+      mkdir kernels
+
+      ${concatStringsSep "\n" (mapAttrsToList (kernelName: kernel:
+        let
+          config = builtins.toJSON {
+            display_name = if (kernel.displayName != "")
+              then kernel.displayName
+              else kernelName;
+            argv = kernel.argv;
+            language = kernel.language;
+          };
+          logo32 =
+            if (kernel.logo32 != null)
+            then "ln -s ${kernel.logo32} 'kernels/${kernelName}/logo-32x32.png';"
+            else "";
+          logo64 =
+            if (kernel.logo64 != null)
+            then "ln -s ${kernel.logo64} 'kernels/${kernelName}/logo-64x64.png';"
+            else "";
+        in ''
+          mkdir 'kernels/${kernelName}';
+          echo '${config}' > 'kernels/${kernelName}/kernel.json';
+          ${logo32}
+          ${logo64}
+        '') definitions)}
+
+      mkdir $out
+      cp -r kernels $out
+    '';
+
+    meta = {
+      description = "Wrapper to create jupyter notebook kernel definitions";
+      homepage = http://jupyter.org/;
+      maintainers = with maintainers; [ aborsu ];
+    };
+  };
+}
diff --git a/pkgs/applications/misc/subsurface/default.nix b/pkgs/applications/misc/subsurface/default.nix
index 5e1e7bf32258..1b998c48580e 100644
--- a/pkgs/applications/misc/subsurface/default.nix
+++ b/pkgs/applications/misc/subsurface/default.nix
@@ -4,14 +4,14 @@
 }:
 
 let
-  version = "4.7.5";
+  version = "4.8.1";
 
   libdc = stdenv.mkDerivation rec {
     name = "libdivecomputer-ssrf-${version}";
 
     src = fetchurl {
       url = "https://subsurface-divelog.org/downloads/libdivecomputer-subsurface-branch-${version}.tgz";
-      sha256 = "1xsgnmgc7yb46lflx8ynkbdxg2f6sny6xg6caqgx7rf0x1jmjj4x";
+      sha256 = "1x6z08gfp9jldv7vcsdasjcarns43qns9cm8s9w27n0c2lvchjvy";
     };
 
     nativeBuildInputs = [ autoreconfHook ];
@@ -70,7 +70,7 @@ in stdenv.mkDerivation rec {
 
   src = fetchurl {
     url = "https://subsurface-divelog.org/downloads/Subsurface-${version}.tgz";
-    sha256 = "0qqmnrmj2alr4rc2nqkv8sbdp92xb6j4j468wn6yqvgb23n77b82";
+    sha256 = "0758sw05gjy8sckvaqc0hmbh2kibmzapgp0hlk8rsp1vsldq4vd2";
   };
 
   buildInputs = [
diff --git a/pkgs/applications/networking/firehol/default.nix b/pkgs/applications/networking/firehol/default.nix
index 9efe20b3ee95..335f393bc077 100644
--- a/pkgs/applications/networking/firehol/default.nix
+++ b/pkgs/applications/networking/firehol/default.nix
@@ -8,13 +8,13 @@
 
 stdenv.mkDerivation rec {
   name = "firehol-${version}";
-  version = "3.1.5";
+  version = "3.1.6";
 
   src = fetchFromGitHub {
     owner = "firehol";
     repo = "firehol";
     rev = "v${version}";
-    sha256 = "15cy1zxfpprma2zkmhj61zzhmw1pfnyhln7pca5lzvr1ifn2d0y0";
+    sha256 = "0l7sjpsb300kqv21hawd26a7jszlmafplacpn5lfj64m4yip93fd";
   };
 
   patches = [
diff --git a/pkgs/applications/networking/instant-messengers/skypeforlinux/default.nix b/pkgs/applications/networking/instant-messengers/skypeforlinux/default.nix
index 9fe03b274cff..d728c8d95cca 100644
--- a/pkgs/applications/networking/instant-messengers/skypeforlinux/default.nix
+++ b/pkgs/applications/networking/instant-messengers/skypeforlinux/default.nix
@@ -6,7 +6,7 @@ let
 
   # Please keep the version x.y.0.z and do not update to x.y.76.z because the
   # source of the latter disappears much faster.
-  version = "8.24.0.2";
+  version = "8.28.0.41";
 
   rpath = stdenv.lib.makeLibraryPath [
     alsaLib
@@ -56,7 +56,7 @@ let
     if stdenv.system == "x86_64-linux" then
       fetchurl {
         url = "https://repo.skype.com/deb/pool/main/s/skypeforlinux/skypeforlinux_${version}_amd64.deb";
-        sha256 = "079bv0wilwwd9gqykcyfs4bj8za140788dxi058k4275h1jlvrww";
+        sha256 = "1kydf71qbz35dx4674h3nxfx8a88k620217906i54ic4qq2mgy2x";
       }
     else
       throw "Skype for linux is not supported on ${stdenv.system}";
diff --git a/pkgs/applications/networking/irc/konversation/default.nix b/pkgs/applications/networking/irc/konversation/default.nix
index c41b6116b29d..ccee78a64da6 100644
--- a/pkgs/applications/networking/irc/konversation/default.nix
+++ b/pkgs/applications/networking/irc/konversation/default.nix
@@ -30,13 +30,13 @@
 
 let
   pname = "konversation";
-  version = "1.7.4";
+  version = "1.7.5";
 in mkDerivation rec {
   name = "${pname}-${version}";
 
   src = fetchurl {
     url = "mirror://kde/stable/${pname}/${version}/src/${name}.tar.xz";
-    sha256 = "0y4rj4fcl1wsi3y1fhnzad9nf4llwrnipfm9mfm55kqnx1zmpvqp";
+    sha256 = "0h098yhlp36ls6pdvs2r93ig8dv4fys62m0h6wxccprb0qrpbgv0";
   };
 
   buildInputs = [
@@ -70,13 +70,6 @@ in mkDerivation rec {
     kdoctools
   ];
 
-  patches = [
-    (fetchpatch {
-      url = "https://cgit.kde.org/konversation.git/patch/?id=20018b3d0798421c9cb8a9d983e5a5b34bd88e8d";
-      sha256 = "0y5m5zimfhc0d1xnkzs05c8ig11lhwdn04fk76vi7966hx8wggnn";
-    })
-  ];
-
   meta = {
     description = "Integrated IRC client for KDE";
     license = with lib.licenses; [ gpl2 ];
diff --git a/pkgs/applications/virtualization/qemu/default.nix b/pkgs/applications/virtualization/qemu/default.nix
index 1ff61129b0f9..f9058ef89b65 100644
--- a/pkgs/applications/virtualization/qemu/default.nix
+++ b/pkgs/applications/virtualization/qemu/default.nix
@@ -22,8 +22,6 @@
 
 with stdenv.lib;
 let
-  version = "2.12.1";
-  sha256 = "1jp5y56682bgpfjapagxjfrjdvqkal34pj9qzn6kj8fqaad80l21";
   audio = optionalString (hasSuffix "linux" stdenv.system) "alsa,"
     + optionalString pulseSupport "pa,"
     + optionalString sdlSupport "sdl,";
@@ -36,6 +34,7 @@ let
 in
 
 stdenv.mkDerivation rec {
+  version = "3.0.0";
   name = "qemu-"
     + stdenv.lib.optionalString xenSupport "xen-"
     + stdenv.lib.optionalString hostCpuOnly "host-cpu-only-"
@@ -43,8 +42,8 @@ stdenv.mkDerivation rec {
     + version;
 
   src = fetchurl {
-    url = "http://wiki.qemu.org/download/qemu-${version}.tar.bz2";
-    inherit sha256;
+    url = "https://wiki.qemu.org/download/qemu-${version}.tar.bz2";
+    sha256 = "1s7bm2xhcxbc9is0rg8xzwijx7azv67skq7mjc58spsgc2nn4glk";
   };
 
   buildInputs =
diff --git a/pkgs/desktops/gnome-3/default.nix b/pkgs/desktops/gnome-3/default.nix
index b4bbdb385de3..d90440d5f557 100644
--- a/pkgs/desktops/gnome-3/default.nix
+++ b/pkgs/desktops/gnome-3/default.nix
@@ -399,6 +399,8 @@ lib.makeScope pkgs.newScope (self: with self; {
 
   gnome-packagekit = callPackage ./misc/gnome-packagekit { };
 
+  # TODO: remove this after 18.09 has forked off
+  gconf = throw "gconf is deprecated since 2009 and has been removed from the package set. Use gnome2.GConf instead. For more details see https://github.com/NixOS/nixpkgs/pull/43268";
 } // lib.optionalAttrs (config.allowAliases or true) {
 #### Legacy aliases
 
diff --git a/pkgs/desktops/gnome-3/games/quadrapassel/default.nix b/pkgs/desktops/gnome-3/games/quadrapassel/default.nix
index b0572f981ded..7ae226b3f8e5 100644
--- a/pkgs/desktops/gnome-3/games/quadrapassel/default.nix
+++ b/pkgs/desktops/gnome-3/games/quadrapassel/default.nix
@@ -2,8 +2,10 @@
 , librsvg, libcanberra-gtk3
 , intltool, itstool, libxml2, clutter, clutter-gtk, wrapGAppsHook }:
 
-stdenv.mkDerivation rec {
-  name = "quadrapassel-${version}";
+let
+  pname = "quadrapassel";
+in stdenv.mkDerivation rec {
+  name = "${pname}-${version}";
   version = "3.22.0";
 
   src = fetchurl {
@@ -11,22 +13,26 @@ stdenv.mkDerivation rec {
     sha256 = "0ed44ef73c8811cbdfc3b44c8fd80eb6e2998d102d59ac324e4748f5d9dddb55";
   };
 
-  passthru = {
-    updateScript = gnome3.updateScript { packageName = "quadrapassel"; attrPath = "gnome3.quadrapassel"; };
-  };
-
-  nativeBuildInputs = [ pkgconfig ];
-  buildInputs = [ gtk3 gnome3.defaultIconTheme gdk_pixbuf librsvg
-                  libcanberra-gtk3 itstool intltool clutter
-                  libxml2 clutter-gtk wrapGAppsHook ];
+  nativeBuildInputs = [ pkgconfig itstool intltool wrapGAppsHook ];
+  buildInputs = [
+    gtk3 gnome3.defaultIconTheme gdk_pixbuf librsvg
+    libcanberra-gtk3 clutter libxml2 clutter-gtk
+  ];
 
   enableParallelBuilding = true;
 
+  passthru = {
+    updateScript = gnome3.updateScript {
+      packageName = pname;
+      attrPath = "gnome3.${pname}";
+    };
+  };
+
   meta = with stdenv.lib; {
-    homepage = https://wiki.gnome.org/Apps/Quadrapassel;
     description = "Classic falling-block game, Tetris";
-    maintainers = gnome3.maintainers;
+    homepage = https://wiki.gnome.org/Apps/Quadrapassel;
     license = licenses.gpl2;
+    maintainers = gnome3.maintainers;
     platforms = platforms.linux;
   };
 }
diff --git a/pkgs/development/libraries/cmrt/default.nix b/pkgs/development/libraries/cmrt/default.nix
new file mode 100644
index 000000000000..8044d7165f75
--- /dev/null
+++ b/pkgs/development/libraries/cmrt/default.nix
@@ -0,0 +1,24 @@
+{ stdenv, fetchurl, autoreconfHook, pkgconfig, libdrm, libva }:
+
+stdenv.mkDerivation rec {
+  name = "cmrt-${version}";
+  version = "1.0.6";
+
+  src = fetchurl {
+    url = "https://github.com/intel/cmrt/archive/${version}.tar.gz";
+    sha256 = "1q7651nvvcqhph5rgfhklm71zqd0c405mrh3wx0cfzvil82yj8na";
+  };
+
+  nativeBuildInputs = [ autoreconfHook pkgconfig ];
+
+  buildInputs = [ libdrm libva ];
+
+  meta = with stdenv.lib; {
+    homepage = https://01.org/linuxmedia;
+    description = "Intel C for Media Runtime";
+    longDescription = "Media GPU kernel manager for Intel G45 & HD Graphics family";
+    license = licenses.mit;
+    maintainers = with maintainers; [ tadfisher ];
+    platforms = platforms.linux;
+  };
+}
diff --git a/pkgs/development/libraries/libmanette/default.nix b/pkgs/development/libraries/libmanette/default.nix
new file mode 100644
index 000000000000..51d2e49eb350
--- /dev/null
+++ b/pkgs/development/libraries/libmanette/default.nix
@@ -0,0 +1,36 @@
+{ stdenv, fetchurl, ninja, meson, pkgconfig, vala, gobjectIntrospection
+, glib, libgudev, libevdev, gnome3 }:
+
+let
+  version = "0.2.1";
+  pname = "libmanette";
+in
+stdenv.mkDerivation {
+  name = "${pname}-${version}";
+
+  outputs = [ "out" "dev" ];
+
+  src = fetchurl {
+    url = "mirror://gnome/sources/${pname}/${stdenv.lib.versions.majorMinor version}/${pname}-${version}.tar.xz";
+    sha256 = "14vqz30p4693yy3yxs0gj858x25sl2kawib1g9lj8g5frgl0hd82";
+  };
+
+  nativeBuildInputs = [ meson ninja pkgconfig vala gobjectIntrospection ];
+  buildInputs = [ glib libgudev libevdev ];
+
+  doCheck = true;
+
+  passthru = {
+    updateScript = gnome3.updateScript {
+      packageName = pname;
+    };
+  };
+
+  meta = with stdenv.lib; {
+    description = "A simple GObject game controller library";
+    homepage = https://wiki.gnome.org/Apps/Builder;
+    license = licenses.lgpl21Plus;
+    maintainers = gnome3.maintainers;
+    platforms = platforms.unix;
+  };
+}
diff --git a/pkgs/development/libraries/mac/default.nix b/pkgs/development/libraries/mac/default.nix
new file mode 100644
index 000000000000..8a8a004a5e69
--- /dev/null
+++ b/pkgs/development/libraries/mac/default.nix
@@ -0,0 +1,36 @@
+{ stdenv, fetchurl, fetchpatch, yasm }:
+
+stdenv.mkDerivation rec {
+  name = "mac-${version}";
+  version = "4.11-u4-b5-s7";
+
+  src = fetchurl {
+    url = "https://www.deb-multimedia.org/pool/main/m/monkeys-audio/monkeys-audio_${version}.orig.tar.gz";
+    sha256 = "16i96cw5r3xbsivjigqp15vv32wa38k86mxq11qx1pzmpryqpqkk";
+  };
+
+  patches = [
+    (fetchpatch {
+      name = "mac-4.11.4.5.7-gcc6.patch";
+      url = "https://gitweb.gentoo.org/repo/gentoo.git/plain/media-sound/mac/files/mac-4.11.4.5.7-gcc6.patch?id=1bd4e0e30e4d8a8862217d7067323851b34c7fe4";
+      sha256 = "093b8m8p8s6dmc62fc8vb4hlmjc2ncb4rdgc82g0a8gg6w5kcj8x";
+    })
+    (fetchpatch {
+      name = "mac-4.11.4.5.7-output.patch";
+      url = "https://gitweb.gentoo.org/repo/gentoo.git/plain/media-sound/mac/files/mac-4.11.4.5.7-output.patch?id=1bd4e0e30e4d8a8862217d7067323851b34c7fe4";
+      sha256 = "0njmwj6d9jqi4pz4fax02w37gk22vda0grszrs2nn97zzmjl36zk";
+    })
+  ];
+
+  CXXFLAGS = "-DSHNTOOL";
+
+  nativeBuildInputs = [ yasm ];
+
+  meta = with stdenv.lib; {
+    description = "APE codec and decompressor";
+    homepage = http://www.deb-multimedia.org/dists/testing/main/binary-amd64/package/monkeys-audio.php;
+    license = licenses.unfreeRedistributable;
+    platforms = [ "x86_64-linux" ];
+    maintainers = with maintainers; [ jfrankenau ];
+  };
+}
diff --git a/pkgs/development/libraries/qt-5/modules/qtwebengine.nix b/pkgs/development/libraries/qt-5/modules/qtwebengine.nix
index 622c0554967d..aae15c62d73c 100644
--- a/pkgs/development/libraries/qt-5/modules/qtwebengine.nix
+++ b/pkgs/development/libraries/qt-5/modules/qtwebengine.nix
@@ -134,7 +134,7 @@ EOF
     harfbuzz icu
 
     libevent
-  ] ++ optionals stdenv.hostPlatform.isArm [
+  ] ++ optionals (stdenv.hostPlatform.isAarch32 || stdenv.hostPlatform.isAarch64) [
     ffmpeg
   ] ++ optionals (!stdenv.isDarwin) [
     dbus zlib minizip snappy nss protobuf jsoncpp
diff --git a/pkgs/development/libraries/vaapi-intel-hybrid/default.nix b/pkgs/development/libraries/vaapi-intel-hybrid/default.nix
new file mode 100644
index 000000000000..56ded2c56447
--- /dev/null
+++ b/pkgs/development/libraries/vaapi-intel-hybrid/default.nix
@@ -0,0 +1,47 @@
+{ stdenv, fetchurl, autoreconfHook, pkgconfig, cmrt, libdrm, libva, libX11, libGL, wayland }:
+
+stdenv.mkDerivation rec {
+  name = "intel-hybrid-driver-${version}";
+  version = "1.0.2";
+
+  src = fetchurl {
+    url = "https://github.com/01org/intel-hybrid-driver/archive/${version}.tar.gz";
+    sha256 = "0ywdhbvzwzzrq4qhylnw1wc8l3j67h26l0cs1rncwhw05s3ndk8n";
+  };
+
+  patches = [
+    # driver_init: load libva-x11.so for any ABI version
+    (fetchurl {
+      url = https://github.com/01org/intel-hybrid-driver/pull/26.diff;
+      sha256 = "1ql4mbi5x1d2a5c8mkjvciaq60zj8nhx912992winbhfkyvpb3gx";
+    })
+  ];
+
+  nativeBuildInputs = [ autoreconfHook pkgconfig ];
+
+  buildInputs = [ cmrt libdrm libva libX11 libGL wayland ];
+
+  enableParallelBuilding = true;
+
+  configureFlags = [
+    "--enable-drm"
+    "--enable-x11"
+    "--enable-wayland"
+  ];
+
+  postPatch = ''
+    patchShebangs ./src/shaders/gpp.py
+  '';
+
+  preConfigure = ''
+    sed -i -e "s,LIBVA_DRIVERS_PATH=.*,LIBVA_DRIVERS_PATH=$out/lib/dri," configure
+  '';
+
+  meta = with stdenv.lib; {
+    homepage = https://01.org/linuxmedia;
+    description = "Intel driver for the VAAPI library with partial HW acceleration";
+    license = licenses.mit;
+    maintainers = with maintainers; [ tadfisher ];
+    platforms = platforms.linux;
+  };
+}
diff --git a/pkgs/development/libraries/vaapi-intel/default.nix b/pkgs/development/libraries/vaapi-intel/default.nix
index 4e729db4671e..ba763e334905 100644
--- a/pkgs/development/libraries/vaapi-intel/default.nix
+++ b/pkgs/development/libraries/vaapi-intel/default.nix
@@ -1,5 +1,6 @@
 { stdenv, fetchFromGitHub, autoreconfHook, gnum4, pkgconfig, python2
 , intel-gpu-tools, libdrm, libva, libX11, libGL, wayland, libXext
+, enableHybridCodec ? false, vaapi-intel-hybrid
 }:
 
 stdenv.mkDerivation rec {
@@ -7,8 +8,8 @@ stdenv.mkDerivation rec {
   inherit (libva) version;
 
   src = fetchFromGitHub {
-    owner  = "01org";
-    repo   = "libva-intel-driver";
+    owner  = "intel";
+    repo   = "intel-vaapi-driver";
     rev    = version;
     sha256 = "15ag4al9h6b8f8sw1zpighyhsmr5qfqp1882q7r3gsh5g4cnj763";
   };
@@ -21,20 +22,25 @@ stdenv.mkDerivation rec {
     sed -i -e "s,LIBVA_DRIVERS_PATH=.*,LIBVA_DRIVERS_PATH=$out/lib/dri," configure
   '';
 
+  postInstall = stdenv.lib.optionalString enableHybridCodec ''
+    ln -s ${vaapi-intel-hybrid}/lib/dri/* $out/lib/dri/
+  '';
+
   configureFlags = [
     "--enable-drm"
     "--enable-x11"
     "--enable-wayland"
-  ];
+  ] ++ stdenv.lib.optional enableHybridCodec "--enable-hybrid-codec";
 
   nativeBuildInputs = [ autoreconfHook gnum4 pkgconfig python2 ];
 
-  buildInputs = [ intel-gpu-tools libdrm libva libX11 libXext libGL wayland ];
+  buildInputs = [ intel-gpu-tools libdrm libva libX11 libXext libGL wayland ]
+    ++ stdenv.lib.optional enableHybridCodec vaapi-intel-hybrid;
 
   enableParallelBuilding = true;
 
   meta = with stdenv.lib; {
-    homepage = https://cgit.freedesktop.org/vaapi/intel-driver/;
+    homepage = https://01.org/linuxmedia;
     license = licenses.mit;
     description = "Intel driver for the VAAPI library";
     platforms = platforms.unix;
diff --git a/pkgs/development/python-modules/elasticsearch-curator/default.nix b/pkgs/development/python-modules/elasticsearch-curator/default.nix
index 4b0aba45cd7a..1ea6e4cabad8 100644
--- a/pkgs/development/python-modules/elasticsearch-curator/default.nix
+++ b/pkgs/development/python-modules/elasticsearch-curator/default.nix
@@ -1,8 +1,10 @@
 { stdenv
 , buildPythonPackage
 , fetchPypi
+, boto3
 , click
 , certifi
+, requests-aws4auth
 , voluptuous
 , pyyaml
 , elasticsearch
@@ -22,17 +24,17 @@ buildPythonPackage rec {
     sha256 = "e75abeb7f7be939b1c64c071898760dc10ab5f08307c253fc074abf8a41a76f0";
   };
 
-  # The integration tests require a running elasticsearch cluster.
-  postUnpackPhase = ''
-    rm -r test/integration
-  '';
+  # The test hangs so we disable it.
+  doCheck = false;
 
   propagatedBuildInputs = [
     click
     certifi
+    requests-aws4auth
     voluptuous
     pyyaml
     elasticsearch
+    boto3
   ];
 
   checkInputs = [
diff --git a/pkgs/development/python-modules/requests-aws4auth/default.nix b/pkgs/development/python-modules/requests-aws4auth/default.nix
new file mode 100644
index 000000000000..b7010eccf0b3
--- /dev/null
+++ b/pkgs/development/python-modules/requests-aws4auth/default.nix
@@ -0,0 +1,28 @@
+{ lib, buildPythonPackage, fetchPypi, fetchzip, isPy3k, requests }:
+with lib;
+buildPythonPackage rec {
+  pname = "requests-aws4auth";
+  version = "0.9";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "0g52a1pm53aqkc9qb5q1m918c1qy6q47c1qz63p5ilynfbs3m5y9";
+  };
+
+  postPatch = optionalString isPy3k ''
+    sed "s/path_encoding_style/'path_encoding_style'/" \
+      -i requests_aws4auth/service_parameters.py
+  '';
+
+  propagatedBuildInputs = [ requests ];
+
+  # The test fail on Python >= 3 because of module import errors.
+  doCheck = !isPy3k;
+
+  meta = {
+    description = "Amazon Web Services version 4 authentication for the Python Requests library.";
+    homepage = https://github.com/sam-washington/requests-aws4auth;
+    license = licenses.mit;
+    maintainers = [ maintainers.basvandijk ];
+  };
+}
diff --git a/pkgs/development/tools/misc/tokei/default.nix b/pkgs/development/tools/misc/tokei/default.nix
index 0059c10afe0c..831a9ec62780 100644
--- a/pkgs/development/tools/misc/tokei/default.nix
+++ b/pkgs/development/tools/misc/tokei/default.nix
@@ -2,16 +2,16 @@
 
 rustPlatform.buildRustPackage rec {
   name = "tokei-${version}";
-  version = "7.0.3";
+  version = "8.0.0";
 
   src = fetchFromGitHub {
     owner = "Aaronepower";
     repo = "tokei";
     rev = "v${version}";
-    sha256 = "1n2ddwmyd414p6a98khq8y0bmljwcclw30wy5zy5b6z1l40yxcza";
+    sha256 = "1sfwmjlvjrd8r0ynnayw7g3514mfiky2j30byphaagdw4jkxbd7c";
   };
 
-  cargoSha256 = "0ngqk8nnw00s25y91vgsl37j26xl2ws75l3lvklf9gbd4bi01crv";
+  cargoSha256 = "0v29gych757h7vv5jsg7rpl705gpqn0ya8ai53582qd2cc6yz4c3";
 
   meta = with stdenv.lib; {
     description = "Count code, quickly";
diff --git a/pkgs/games/20kly/default.nix b/pkgs/games/20kly/default.nix
new file mode 100644
index 000000000000..37cb96f2cfdb
--- /dev/null
+++ b/pkgs/games/20kly/default.nix
@@ -0,0 +1,40 @@
+{ stdenv
+, fetchurl
+, python }:
+
+python.pkgs.buildPythonApplication rec {
+  pname = "20kly";
+  version = "1.4";
+  format = "other";
+  disabled = !(python.isPy2 or false);
+
+  src = fetchurl {
+    url = "http://jwhitham.org.uk/20kly/lightyears-${version}.tar.bz2";
+    sha256 = "13h73cmfjqkipffimfc4iv0hf89if490ng6vd6xf3wcalpgaim5d";
+  };
+
+  patchPhase = ''
+    substituteInPlace lightyears \
+      --replace \
+        "LIGHTYEARS_DIR = \".\"" \
+        "LIGHTYEARS_DIR = \"$out/share\""
+  '';
+
+  propagatedBuildInputs = with python.pkgs; [ pygame ];
+
+  buildPhase = "python -O -m compileall .";
+
+  installPhase = ''
+    mkdir -p "$out/share"
+    cp -r audio code data lightyears "$out/share"
+    install -Dm755 lightyears "$out/bin/lightyears"
+  '';
+
+  meta = with stdenv.lib; {
+    description = "A steampunk-themed strategy game where you have to manage a steam supply network";
+    homepage = http://jwhitham.org.uk/20kly/;
+    license = licenses.gpl2;
+    maintainers = with maintainers; [ fgaz ];
+  };
+}
+
diff --git a/pkgs/os-specific/linux/firmware/fwupdate/default.nix b/pkgs/os-specific/linux/firmware/fwupdate/default.nix
index fd0d222eaac8..5b14546c90fd 100644
--- a/pkgs/os-specific/linux/firmware/fwupdate/default.nix
+++ b/pkgs/os-specific/linux/firmware/fwupdate/default.nix
@@ -1,16 +1,14 @@
 { efivar, fetchurl, gettext, gnu-efi, libsmbios, pkgconfig, popt, stdenv }:
 let
-  version = "10";
+  version = "12";
 in stdenv.mkDerivation {
   name = "fwupdate-${version}";
   src = fetchurl {
     url = "https://github.com/rhinstaller/fwupdate/releases/download/${version}/fwupdate-${version}.tar.bz2";
-    sha256 = "0fpk3q0msq2l0bs2mvk0cqp8jbwnmi17ggc81r4v96h4jxh2rx3k";
+    sha256 = "00w7jsg7wrlq4cpfz26m9rbv2jwyf0sansf343vfq02fy5lxars1";
   };
 
   patches = [
-    # https://github.com/rhboot/fwupdate/pull/99
-    ./fix-paths.patch
     ./do-not-create-sharedstatedir.patch
   ];
 
diff --git a/pkgs/os-specific/linux/firmware/fwupdate/fix-paths.patch b/pkgs/os-specific/linux/firmware/fwupdate/fix-paths.patch
deleted file mode 100644
index b01dfa355ef7..000000000000
--- a/pkgs/os-specific/linux/firmware/fwupdate/fix-paths.patch
+++ /dev/null
@@ -1,116 +0,0 @@
---- a/Make.defaults
-+++ b/Make.defaults
-@@ -18,19 +18,20 @@
- ABIDIFF		:= abidiff
- ABIDW		:= abidw
- prefix		?= /usr/
--prefix		:= $(abspath $(prefix))/
-+prefix		:= $(abspath $(prefix))
-+exec_prefix		?= $(prefix)
- ARCH		= $(shell $(CC) -dumpmachine | cut -f1 -d- | sed s,i[3456789]86,ia32,)
- ifeq ($(ARCH),x86_64)
--  LIBDIR	?= $(prefix)lib64
-+  LIBDIR	?= $(exec_prefix)/lib64
- endif
- ifeq ($(ARCH),ia32)
--  LIBDIR	?= $(prefix)lib
-+  LIBDIR	?= $(exec_prefix)/lib
- endif
- ifeq ($(ARCH),aarch64)
--  LIBDIR	?= $(prefix)lib64
-+  LIBDIR	?= $(exec_prefix)/lib64
- endif
- ifeq ($(ARCH),arm)
--  LIBDIR	?= $(prefix)lib
-+  LIBDIR	?= $(exec_prefix)/lib
- endif
- LIBDIR		?= unknown
- ifeq ($(LIBDIR),unknown)
-@@ -45,13 +46,13 @@
- GNUEFIDIR	?= $(LIBDIR)/gnuefi
- libdir		?= $(LIBDIR)
- pcdir		?= $(libdir)/pkgconfig
--mandir		?= $(prefix)share/man
--includedir	?= $(prefix)include
--bindir		?= $(prefix)bin
--datadir 	?= $(prefix)share
-+mandir		?= $(prefix)/share/man
-+includedir	?= $(prefix)/include
-+bindir		?= $(exec_prefix)/bin
-+datadir 	?= $(prefix)/share
- localedir	?= $(datadir)/locale
--libexecdir	?= $(prefix)libexec
--libdatadir	?= $(prefix)lib
-+libexecdir	?= $(exec_prefix)/libexec
-+libdatadir	?= $(exec_prefix)/lib
- sharedstatedir	?= /var/lib
- 
- EFIDIR		?= $(shell x=$$(which --skip-alias --skip-functions git 2>/dev/null) ; [ -n "$$x" ] && git config --get fwupdate.efidir)
-@@ -63,8 +64,8 @@
- 	ESPMOUNTPOINT = "/boot/efi"
- endif
- 
--DEBUGINFO	?= $(prefix)lib/debug
--DEBUGSOURCE	?= $(prefix)src/debug
-+DEBUGINFO	?= $(exec_prefix)/lib/debug
-+DEBUGSOURCE	?= $(prefix)/src/debug
- TARGETDIR	?= $(ESPMOUNTPOINT)/EFI/$(EFIDIR)
- 
- .PHONY: check_efidir_error
---- a/linux/Makefile
-+++ b/linux/Makefile
-@@ -93,16 +93,19 @@
- 	sed -e "s,@@VERSION@@,$(VERSION),g" \
- 		-e "s,@@FWUP_MAJOR_VERSION@@,$(MAJOR_VERSION),g" \
- 		-e "s,@@FWUP_MINOR_VERSION@@,$(MINOR_VERSION),g" \
-+		-e "s,@@PREFIX@@,$(prefix),g" \
-+		-e "s,@@EXEC_PREFIX@@,$(exec_prefix),g" \
- 		-e "s,@@SHAREDSTATEDIR@@,$(sharedstatedir),g" \
- 		-e "s,@@ESPMOUNTPOINT@@,$(ESPMOUNTPOINT),g" \
- 		-e "s,@@EFIDIR@@,$(EFIDIR),g" \
- 		-e "s,@@LIBDIR@@,$(libdir),g" \
- 		-e "s,@@LIBEXECDIR@@,$(libexecdir),g" \
-+		-e "s,@@INCLUDEDIR@@,$(includedir),g" \
- 		$< > $@
- 
- tester : tester.c
--	$(CC) -Wall -Werror -ggdb -L. -I./include -I/usr/include/efivar \
--		-lefivar -lfwup -o $@ $^
-+	$(CC) -Wall -Werror -ggdb -L. -I./include $(shell $(PKG_CONFIG) --cflags efivar) \
-+		$(shell $(PKG_CONFIG) --libs efivar) -lfwup -o $@ $^
- 
- test : tester
- 	LD_LIBRARY_PATH=$(shell pwd) ./tester
-@@ -134,6 +137,6 @@
- 	$(INSTALL) -d -m 755 $(DESTDIR)$(libdatadir)/systemd/system
- 	$(INSTALL) -m 644 cleanup.service \
- 		$(DESTDIR)$(libdatadir)/systemd/system/fwupdate-cleanup.service
--	$(INSTALL) -d -m 755 $(DESTDIR)/usr/share/bash-completion/completions/
-+	$(INSTALL) -d -m 755 $(DESTDIR)$(datadir)/bash-completion/completions/
- 	$(INSTALL) -m 755 $(VPATH)/bash-completion \
--		$(DESTDIR)/usr/share/bash-completion/completions/fwupdate
-+		$(DESTDIR)$(datadir)/bash-completion/completions/fwupdate
---- a/linux/fwup.pc.in
-+++ b/linux/fwup.pc.in
-@@ -1,7 +1,7 @@
--prefix=/usr
--exec_prefix=/usr
-+prefix=@@PREFIX@@
-+exec_prefix=@@EXEC_PREFIX@@
- libdir=@@LIBDIR@@
--includedir=/usr/include
-+includedir=@@INCLUDEDIR@@
- 
- Name: fwup
- Description: Library for deployment of UEFI firmware updates.
---- a/linux/libfwup.c
-+++ b/linux/libfwup.c
-@@ -35,7 +35,7 @@
- 
- #include <dell-wmi-smi.h>
- #ifdef FWUPDATE_HAVE_LIBSMBIOS__
--#include </usr/include/smbios_c/token.h>
-+#include <smbios_c/token.h>
- #include <smbios_c/smi.h>
- #endif
- 
diff --git a/pkgs/os-specific/linux/kernel/common-config.nix b/pkgs/os-specific/linux/kernel/common-config.nix
index 548c66e87e63..dd3186bc8760 100644
--- a/pkgs/os-specific/linux/kernel/common-config.nix
+++ b/pkgs/os-specific/linux/kernel/common-config.nix
@@ -146,6 +146,14 @@ let
 
       # needed for iwd WPS support (wpa_supplicant replacement)
       KEY_DH_OPERATIONS = whenAtLeast "4.7" yes;
+
+      # needed for nftables
+      NF_TABLES_INET              = whenAtLeast "4.17" yes;
+      NF_TABLES_NETDEV            = whenAtLeast "4.17" yes;
+      NF_TABLES_IPV4              = whenAtLeast "4.17" yes;
+      NF_TABLES_ARP               = whenAtLeast "4.17" yes;
+      NF_TABLES_IPV6              = whenAtLeast "4.17" yes;
+      NF_TABLES_BRIDGE            = whenAtLeast "4.17" yes;
     };
 
     wireless = {
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 6a25ec7c561f..4d2bf622e44e 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -3383,6 +3383,10 @@ with pkgs;
 
   jupp = callPackage ../applications/editors/jupp { };
 
+  jupyter = callPackage ../applications/editors/jupyter { };
+
+  jupyter-kernel = callPackage ../applications/editors/jupyter/kernel.nix { };
+
   jwhois = callPackage ../tools/networking/jwhois { };
 
   k2pdfopt = callPackage ../applications/misc/k2pdfopt { };
@@ -9034,6 +9038,8 @@ with pkgs;
 
   cmocka = callPackage ../development/libraries/cmocka { };
 
+  cmrt = callPackage ../development/libraries/cmrt { };
+
   cogl = callPackage ../development/libraries/cogl { };
 
   coin3d = callPackage ../development/libraries/coin3d { };
@@ -10623,6 +10629,8 @@ with pkgs;
 
   libmad = callPackage ../development/libraries/libmad { };
 
+  libmanette = callPackage ../development/libraries/libmanette { };
+
   libmatchbox = callPackage ../development/libraries/libmatchbox { };
 
   libmatheval = callPackage ../development/libraries/libmatheval {
@@ -12276,6 +12284,8 @@ with pkgs;
 
   vaapiIntel = callPackage ../development/libraries/vaapi-intel { };
 
+  vaapi-intel-hybrid = callPackage ../development/libraries/vaapi-intel-hybrid { };
+
   vaapiVdpau = callPackage ../development/libraries/vaapi-vdpau { };
 
   vale = callPackage ../tools/text/vale { };
@@ -17320,6 +17330,8 @@ with pkgs;
 
   lyx = libsForQt5.callPackage ../applications/misc/lyx { };
 
+  mac = callPackage ../development/libraries/mac { };
+
   magic-wormhole = with python3Packages; toPythonApplication magic-wormhole;
 
   mail-notification = callPackage ../desktops/gnome-2/desktop/mail-notification {};
@@ -18423,6 +18435,8 @@ with pkgs;
 
   spideroak = callPackage ../applications/networking/spideroak { };
 
+  split2flac = callPackage ../applications/audio/split2flac { };
+
   squishyball = callPackage ../applications/audio/squishyball {
     ncurses = ncurses5;
   };
@@ -19562,6 +19576,8 @@ with pkgs;
 
   _2048-in-terminal = callPackage ../games/2048-in-terminal { };
 
+  _20kly = callPackage ../games/20kly { };
+
   _90secondportraits = callPackage ../games/90secondportraits { love = love_0_10; };
 
   adom = callPackage ../games/adom { };
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 322f831b4b50..3ba335146ce3 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -2009,6 +2009,8 @@ in {
 
   requests-unixsocket = callPackage ../development/python-modules/requests-unixsocket {};
 
+  requests-aws4auth = callPackage ../development/python-modules/requests-aws4auth { };
+
   howdoi = callPackage ../development/python-modules/howdoi {};
 
   neurotools = callPackage ../development/python-modules/neurotools {};