diff options
author | Bas van Dijk <v.dijk.bas@gmail.com> | 2018-08-26 01:33:34 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-08-26 01:33:34 +0200 |
commit | a144c798e561db99b9bddca562ffb4c3645efe5f (patch) | |
tree | 1027e4955f6a6307a16b56a32794a4e89cbb1713 | |
parent | b2dc75cd03dc6254a4df6d7fb8e3b41989f05a65 (diff) | |
parent | 954eb34a5363877e5efed27ddc2bc748f6265019 (diff) | |
download | nixlib-a144c798e561db99b9bddca562ffb4c3645efe5f.tar nixlib-a144c798e561db99b9bddca562ffb4c3645efe5f.tar.gz nixlib-a144c798e561db99b9bddca562ffb4c3645efe5f.tar.bz2 nixlib-a144c798e561db99b9bddca562ffb4c3645efe5f.tar.lz nixlib-a144c798e561db99b9bddca562ffb4c3645efe5f.tar.xz nixlib-a144c798e561db99b9bddca562ffb4c3645efe5f.tar.zst nixlib-a144c798e561db99b9bddca562ffb4c3645efe5f.zip |
Merge pull request #44340 from shmish111/es-curator
nixos/curator: init elasticsearch curator
-rw-r--r-- | nixos/doc/manual/release-notes/rl-1809.xml | 6 | ||||
-rw-r--r-- | nixos/modules/module-list.nix | 1 | ||||
-rw-r--r-- | nixos/modules/services/search/elasticsearch-curator.nix | 93 | ||||
-rw-r--r-- | nixos/tests/elk.nix | 32 | ||||
-rw-r--r-- | pkgs/development/python-modules/elasticsearch-curator/default.nix | 10 | ||||
-rw-r--r-- | pkgs/development/python-modules/requests-aws4auth/default.nix | 28 | ||||
-rw-r--r-- | pkgs/top-level/python-packages.nix | 2 |
7 files changed, 168 insertions, 4 deletions
diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml index 8ee2a5f16238..d190394b9887 100644 --- a/nixos/doc/manual/release-notes/rl-1809.xml +++ b/nixos/doc/manual/release-notes/rl-1809.xml @@ -111,6 +111,12 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull' <link xlink:href="https://github.com/strongswan/strongswan/blob/master/README_LEGACY.md">stroke configuration interface</link>. </para> </listitem> + <listitem> + <para> + The new <varname>services.elasticsearch-curator</varname> service + periodically curates or manages, your Elasticsearch indices and snapshots. + </para> + </listitem> </itemizedlist> </section> diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 2846afea8fbc..85440a8025c9 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -623,6 +623,7 @@ ./services/scheduling/fcron.nix ./services/scheduling/marathon.nix ./services/search/elasticsearch.nix + ./services/search/elasticsearch-curator.nix ./services/search/hound.nix ./services/search/kibana.nix ./services/search/solr.nix diff --git a/nixos/modules/services/search/elasticsearch-curator.nix b/nixos/modules/services/search/elasticsearch-curator.nix new file mode 100644 index 000000000000..43785c392fee --- /dev/null +++ b/nixos/modules/services/search/elasticsearch-curator.nix @@ -0,0 +1,93 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.elasticsearch-curator; + curatorConfig = pkgs.writeTextFile { + name = "config.yaml"; + text = '' + --- + # Remember, leave a key empty if there is no value. None will be a string, + # not a Python "NoneType" + client: + hosts: ${builtins.toJSON cfg.hosts} + port: ${toString cfg.port} + url_prefix: + use_ssl: False + certificate: + client_cert: + client_key: + ssl_no_validate: False + http_auth: + timeout: 30 + master_only: False + logging: + loglevel: INFO + logfile: + logformat: default + blacklist: ['elasticsearch', 'urllib3'] + ''; + }; + curatorAction = pkgs.writeTextFile { + name = "action.yaml"; + text = cfg.actionYAML; + }; +in { + + options.services.elasticsearch-curator = { + + enable = mkEnableOption "elasticsearch curator"; + interval = mkOption { + description = "The frequency to run curator, a systemd.time such as 'hourly'"; + default = "hourly"; + type = types.str; + }; + hosts = mkOption { + description = "a list of elasticsearch hosts to connect to"; + type = types.listOf types.str; + default = ["localhost"]; + }; + port = mkOption { + description = "the port that elasticsearch is listening on"; + type = types.int; + default = 9200; + }; + actionYAML = mkOption { + description = "curator action.yaml file contents, alternatively use curator-cli which takes a simple action command"; + example = '' + --- + actions: + 1: + action: delete_indices + description: >- + Delete indices older than 45 days (based on index name), for logstash- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: 45 + ''; + }; + }; + + config = mkIf cfg.enable { + + systemd.services.elasticsearch-curator = { + startAt = cfg.interval; + serviceConfig = { + ExecStart = ''${pkgs.python36Packages.elasticsearch-curator}/bin/curator --config ${curatorConfig} ${curatorAction}''; + }; + }; + }; +} diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 4c5c441ca265..15be72b80bba 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -63,6 +63,33 @@ let package = elk.kibana; elasticsearch.url = esUrl; }; + + elasticsearch-curator = { + enable = true; + actionYAML = '' + --- + actions: + 1: + action: delete_indices + description: >- + Delete indices older than 1 second (based on index name), for logstash- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: seconds + unit_count: 1 + ''; + }; }; }; }; @@ -91,6 +118,11 @@ let # See if logstash messages arive in elasticsearch. $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); + + # Test elasticsearch-curator. + $one->systemctl("stop logstash"); + $one->systemctl("start elasticsearch-curator"); + $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1"); ''; }; in mapAttrs mkElkTest { diff --git a/pkgs/development/python-modules/elasticsearch-curator/default.nix b/pkgs/development/python-modules/elasticsearch-curator/default.nix index 4b0aba45cd7a..1ea6e4cabad8 100644 --- a/pkgs/development/python-modules/elasticsearch-curator/default.nix +++ b/pkgs/development/python-modules/elasticsearch-curator/default.nix @@ -1,8 +1,10 @@ { stdenv , buildPythonPackage , fetchPypi +, boto3 , click , certifi +, requests-aws4auth , voluptuous , pyyaml , elasticsearch @@ -22,17 +24,17 @@ buildPythonPackage rec { sha256 = "e75abeb7f7be939b1c64c071898760dc10ab5f08307c253fc074abf8a41a76f0"; }; - # The integration tests require a running elasticsearch cluster. - postUnpackPhase = '' - rm -r test/integration - ''; + # The test hangs so we disable it. + doCheck = false; propagatedBuildInputs = [ click certifi + requests-aws4auth voluptuous pyyaml elasticsearch + boto3 ]; checkInputs = [ diff --git a/pkgs/development/python-modules/requests-aws4auth/default.nix b/pkgs/development/python-modules/requests-aws4auth/default.nix new file mode 100644 index 000000000000..b7010eccf0b3 --- /dev/null +++ b/pkgs/development/python-modules/requests-aws4auth/default.nix @@ -0,0 +1,28 @@ +{ lib, buildPythonPackage, fetchPypi, fetchzip, isPy3k, requests }: +with lib; +buildPythonPackage rec { + pname = "requests-aws4auth"; + version = "0.9"; + + src = fetchPypi { + inherit pname version; + sha256 = "0g52a1pm53aqkc9qb5q1m918c1qy6q47c1qz63p5ilynfbs3m5y9"; + }; + + postPatch = optionalString isPy3k '' + sed "s/path_encoding_style/'path_encoding_style'/" \ + -i requests_aws4auth/service_parameters.py + ''; + + propagatedBuildInputs = [ requests ]; + + # The test fail on Python >= 3 because of module import errors. + doCheck = !isPy3k; + + meta = { + description = "Amazon Web Services version 4 authentication for the Python Requests library."; + homepage = https://github.com/sam-washington/requests-aws4auth; + license = licenses.mit; + maintainers = [ maintainers.basvandijk ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 322f831b4b50..3ba335146ce3 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2009,6 +2009,8 @@ in { requests-unixsocket = callPackage ../development/python-modules/requests-unixsocket {}; + requests-aws4auth = callPackage ../development/python-modules/requests-aws4auth { }; + howdoi = callPackage ../development/python-modules/howdoi {}; neurotools = callPackage ../development/python-modules/neurotools {}; |