nixos/hadoop: add HBase submodule

3 files changed, 202 insertions, 1 deletions

diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix
index e3c26a0d5505..874e3227b74c 100644
--- a/nixos/modules/services/cluster/hadoop/conf.nix
+++ b/nixos/modules/services/cluster/hadoop/conf.nix
@@ -33,6 +33,7 @@ pkgs.runCommand "hadoop-conf" {} (with cfg; ''
   mkdir -p $out/
   cp ${siteXml "core-site.xml" (coreSite // coreSiteInternal)}/* $out/
   cp ${siteXml "hdfs-site.xml" (hdfsSiteDefault // hdfsSite // hdfsSiteInternal)}/* $out/
+  cp ${siteXml "hbase-site.xml" (hbaseSiteDefault // hbaseSite // hbaseSiteInternal)}/* $out/
   cp ${siteXml "mapred-site.xml" (mapredSiteDefault // mapredSite)}/* $out/
   cp ${siteXml "yarn-site.xml" (yarnSiteDefault // yarnSite // yarnSiteInternal)}/* $out/
   cp ${siteXml "httpfs-site.xml" httpfsSite}/* $out/
diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix
index a4fdea81037c..a3a28eb7184a 100644
--- a/nixos/modules/services/cluster/hadoop/default.nix
+++ b/nixos/modules/services/cluster/hadoop/default.nix
@@ -5,7 +5,7 @@ let
 in
 with lib;
 {
-  imports = [ ./yarn.nix ./hdfs.nix ];
+  imports = [ ./yarn.nix ./hdfs.nix ./hbase.nix ];
 
   options.services.hadoop = {
     coreSite = mkOption {
diff --git a/nixos/modules/services/cluster/hadoop/hbase.nix b/nixos/modules/services/cluster/hadoop/hbase.nix
new file mode 100644
index 000000000000..050bcd03749d
--- /dev/null
+++ b/nixos/modules/services/cluster/hadoop/hbase.nix
@@ -0,0 +1,200 @@
+{ config, lib, pkgs, ...}:
+
+with lib;
+let
+  cfg = config.services.hadoop;
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+  mkIfNotNull = x: mkIf (x != null) x;
+in
+{
+  options.services.hadoop = {
+
+    gatewayRole.enableHbaseCli = mkOption {
+      description = "Whether to enable HBase CLI tools";
+      default = false;
+      type = types.bool;
+    };
+
+    hbaseSiteDefault = mkOption {
+      default = {
+        "hbase.regionserver.ipc.address" = "0.0.0.0";
+        "hbase.master.ipc.address" = "0.0.0.0";
+        "hbase.master.info.bindAddress" = "0.0.0.0";
+        "hbase.regionserver.info.bindAddress" = "0.0.0.0";
+
+        "hbase.cluster.distributed" = "true";
+      };
+      type = types.attrsOf types.anything;
+      description = ''
+        Default options for hbase-site.xml
+      '';
+    };
+    hbaseSite = mkOption {
+      default = {};
+      type = with types; attrsOf anything;
+      example = literalExpression ''
+      '';
+      description = ''
+        Additional options and overrides for hbase-site.xml
+        <link xlink:href="https://github.com/apache/hbase/blob/rel/2.4.11/hbase-common/src/main/resources/hbase-default.xml"/>
+      '';
+    };
+    hbaseSiteInternal = mkOption {
+      default = {};
+      type = with types; attrsOf anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to hbase-site.xml based on module options
+      '';
+    };
+
+    hbase = {
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.hbase;
+        defaultText = literalExpression "pkgs.hbase";
+        description = "HBase package";
+      };
+
+      rootdir = mkOption {
+        description = ''
+          This option will set "hbase.rootdir" in hbase-site.xml and determine
+          the directory shared by region servers and into which HBase persists.
+          The URL should be 'fully-qualified' to include the filesystem scheme.
+          If a core-site.xml is provided, the FS scheme defaults to the value
+          of "fs.defaultFS".
+
+          Filesystems other than HDFS (like S3, QFS, Swift) are also supported.
+        '';
+        type = types.str;
+        example = "hdfs://nameservice1/hbase";
+        default = "/hbase";
+      };
+      zookeeperQuorum = mkOption {
+        description = ''
+          This option will set "hbase.zookeeper.quorum" in hbase-site.xml.
+          Comma separated list of servers in the ZooKeeper ensemble.
+        '';
+        type = with types; nullOr commas;
+        example = "zk1.internal,zk2.internal,zk3.internal";
+        default = null;
+      };
+      master = {
+        enable = mkEnableOption "HBase Master";
+        initHDFS = mkEnableOption "initialization of the hbase directory on HDFS";
+
+        openFirewall = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Open firewall ports for HBase master.
+          '';
+        };
+      };
+      regionServer = {
+        enable = mkEnableOption "HBase RegionServer";
+
+        overrideHosts = mkOption {
+          type = types.bool;
+          default = true;
+          description = ''
+            Remove /etc/hosts entries for "127.0.0.2" and "::1" defined in nixos/modules/config/networking.nix
+            Regionservers must be able to resolve their hostnames to their IP addresses, through PTR records
+            or /etc/hosts entries.
+
+          '';
+        };
+
+        openFirewall = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Open firewall ports for HBase master.
+          '';
+        };
+      };
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.hbase.master.enable {
+      services.hadoop.gatewayRole = {
+        enable = true;
+        enableHbaseCli = mkDefault true;
+      };
+
+      systemd.services.hbase-master = {
+        description = "HBase master";
+        wantedBy = [ "multi-user.target" ];
+
+        preStart = mkIf cfg.hbase.master.initHDFS ''
+          HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfsadmin -safemode wait
+          HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -mkdir -p ${cfg.hbase.rootdir}
+          HADOOP_USER_NAME=hdfs ${cfg.package}/bin/hdfs --config ${hadoopConf} dfs -chown hbase ${cfg.hbase.rootdir}
+        '';
+
+        serviceConfig = {
+          User = "hbase";
+          SyslogIdentifier = "hbase-master";
+          ExecStart = "${cfg.hbase.package}/bin/hbase --config ${hadoopConf} " +
+                      "master start";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
+
+      networking.firewall.allowedTCPPorts = (mkIf cfg.hbase.master.openFirewall [
+        16000 16010
+      ]);
+
+    })
+
+    (mkIf cfg.hbase.regionServer.enable {
+      services.hadoop.gatewayRole = {
+        enable = true;
+        enableHbaseCli = mkDefault true;
+      };
+
+      systemd.services.hbase-regionserver = {
+        description = "HBase RegionServer";
+        wantedBy = [ "multi-user.target" ];
+        serviceConfig = {
+          User = "hbase";
+          SyslogIdentifier = "hbase-regionserver";
+          ExecStart = "${cfg.hbase.package}/bin/hbase --config /etc/hadoop-conf/ " +
+                      "regionserver start";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
+
+      networking = {
+        firewall.allowedTCPPorts = (mkIf cfg.hbase.regionServer.openFirewall [
+          16020 16030
+        ]);
+        hosts = mkIf cfg.hbase.regionServer.overrideHosts {
+          "127.0.0.2" = mkForce [ ];
+          "::1" = mkForce [ ];
+        };
+      };
+    })
+
+    (mkIf cfg.gatewayRole.enable {
+
+      environment.systemPackages = mkIf cfg.gatewayRole.enableHbaseCli [ cfg.hbase.package ];
+
+      services.hadoop.hbaseSiteInternal = with cfg.hbase; {
+        "hbase.zookeeper.quorum" = mkIfNotNull zookeeperQuorum;
+      };
+
+      users.users.hbase = {
+        description = "Hadoop HBase user";
+        group = "hadoop";
+        isSystemUser = true;
+      };
+    })
+  ];
+}