diff options
Diffstat (limited to 'nixos')
| -rw-r--r-- | nixos/doc/manual/release-notes/rl-1809.xml | 6 | ||||
| -rw-r--r-- | nixos/modules/module-list.nix | 1 | ||||
| -rw-r--r-- | nixos/modules/services/search/elasticsearch-curator.nix | 93 | ||||
| -rw-r--r-- | nixos/tests/elk.nix | 32 |
4 files changed, 132 insertions, 0 deletions
diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml index 8ee2a5f16238..d190394b9887 100644 --- a/nixos/doc/manual/release-notes/rl-1809.xml +++ b/nixos/doc/manual/release-notes/rl-1809.xml @@ -111,6 +111,12 @@ $ nix-instantiate -E '(import <nixpkgsunstable> {}).gitFull' <link xlink:href="https://github.com/strongswan/strongswan/blob/master/README_LEGACY.md">stroke configuration interface</link>. </para> </listitem> + <listitem> + <para> + The new <varname>services.elasticsearch-curator</varname> service + periodically curates or manages, your Elasticsearch indices and snapshots. + </para> + </listitem> </itemizedlist> </section> diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 2846afea8fbc..85440a8025c9 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -623,6 +623,7 @@ ./services/scheduling/fcron.nix ./services/scheduling/marathon.nix ./services/search/elasticsearch.nix + ./services/search/elasticsearch-curator.nix ./services/search/hound.nix ./services/search/kibana.nix ./services/search/solr.nix diff --git a/nixos/modules/services/search/elasticsearch-curator.nix b/nixos/modules/services/search/elasticsearch-curator.nix new file mode 100644 index 000000000000..43785c392fee --- /dev/null +++ b/nixos/modules/services/search/elasticsearch-curator.nix @@ -0,0 +1,93 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.services.elasticsearch-curator; + curatorConfig = pkgs.writeTextFile { + name = "config.yaml"; + text = '' + --- + # Remember, leave a key empty if there is no value. None will be a string, + # not a Python "NoneType" + client: + hosts: ${builtins.toJSON cfg.hosts} + port: ${toString cfg.port} + url_prefix: + use_ssl: False + certificate: + client_cert: + client_key: + ssl_no_validate: False + http_auth: + timeout: 30 + master_only: False + logging: + loglevel: INFO + logfile: + logformat: default + blacklist: ['elasticsearch', 'urllib3'] + ''; + }; + curatorAction = pkgs.writeTextFile { + name = "action.yaml"; + text = cfg.actionYAML; + }; +in { + + options.services.elasticsearch-curator = { + + enable = mkEnableOption "elasticsearch curator"; + interval = mkOption { + description = "The frequency to run curator, a systemd.time such as 'hourly'"; + default = "hourly"; + type = types.str; + }; + hosts = mkOption { + description = "a list of elasticsearch hosts to connect to"; + type = types.listOf types.str; + default = ["localhost"]; + }; + port = mkOption { + description = "the port that elasticsearch is listening on"; + type = types.int; + default = 9200; + }; + actionYAML = mkOption { + description = "curator action.yaml file contents, alternatively use curator-cli which takes a simple action command"; + example = '' + --- + actions: + 1: + action: delete_indices + description: >- + Delete indices older than 45 days (based on index name), for logstash- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: days + unit_count: 45 + ''; + }; + }; + + config = mkIf cfg.enable { + + systemd.services.elasticsearch-curator = { + startAt = cfg.interval; + serviceConfig = { + ExecStart = ''${pkgs.python36Packages.elasticsearch-curator}/bin/curator --config ${curatorConfig} ${curatorAction}''; + }; + }; + }; +} diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 4c5c441ca265..15be72b80bba 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -63,6 +63,33 @@ let package = elk.kibana; elasticsearch.url = esUrl; }; + + elasticsearch-curator = { + enable = true; + actionYAML = '' + --- + actions: + 1: + action: delete_indices + description: >- + Delete indices older than 1 second (based on index name), for logstash- + prefixed indices. Ignore the error if the filter does not result in an + actionable list of indices (ignore_empty_list) and exit cleanly. + options: + ignore_empty_list: True + disable_action: False + filters: + - filtertype: pattern + kind: prefix + value: logstash- + - filtertype: age + source: name + direction: older + timestring: '%Y.%m.%d' + unit: seconds + unit_count: 1 + ''; + }; }; }; }; @@ -91,6 +118,11 @@ let # See if logstash messages arive in elasticsearch. $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); + + # Test elasticsearch-curator. + $one->systemctl("stop logstash"); + $one->systemctl("start elasticsearch-curator"); + $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1"); ''; }; in mapAttrs mkElkTest { |