From b299e508eab87b26f3624421a35d76efa026ae3b Mon Sep 17 00:00:00 2001 From: Dan Peebles Date: Tue, 24 Jan 2017 21:21:38 +0000 Subject: fluentd: 0.14.0 -> 0.14.11, with more plugins I added the most popular plugins (S3, kafka, kinesis, mongo, etc.) and tried to bring it closer to the set of plugins they bundle with td-agent, without getting rid of the ones we already have today. We should probably make this overridable more generically going forward but this should scratch most itches today. --- pkgs/tools/misc/fluentd/Gemfile | 9 + pkgs/tools/misc/fluentd/Gemfile.lock | 130 +++++++++++--- pkgs/tools/misc/fluentd/gemset.nix | 319 +++++++++++++++++++++++++++++++---- 3 files changed, 401 insertions(+), 57 deletions(-) (limited to 'pkgs/tools/misc/fluentd') diff --git a/pkgs/tools/misc/fluentd/Gemfile b/pkgs/tools/misc/fluentd/Gemfile index 8c9dd3aa0a02..2c4fbc849631 100644 --- a/pkgs/tools/misc/fluentd/Gemfile +++ b/pkgs/tools/misc/fluentd/Gemfile @@ -3,3 +3,12 @@ source "https://rubygems.org" gem 'fluentd' gem 'fluent-plugin-elasticsearch' gem 'fluent-plugin-record-reformer' +gem 'fluent-plugin-s3' +gem 'fluent-plugin-kinesis' +gem 'fluent-plugin-kafka' +gem 'fluent-plugin-elasticsearch' +gem 'fluent-plugin-scribe' +gem 'fluent-plugin-mongo' +gem 'fluent-plugin-webhdfs' +gem 'fluent-plugin-rewrite-tag-filter' + diff --git a/pkgs/tools/misc/fluentd/Gemfile.lock b/pkgs/tools/misc/fluentd/Gemfile.lock index 581fa6e169ae..2f9485d95777 100644 --- a/pkgs/tools/misc/fluentd/Gemfile.lock +++ b/pkgs/tools/misc/fluentd/Gemfile.lock @@ -1,58 +1,136 @@ GEM remote: https://rubygems.org/ specs: - cool.io (1.4.4) - elasticsearch (1.0.17) - elasticsearch-api (= 1.0.17) - elasticsearch-transport (= 1.0.17) - elasticsearch-api (1.0.17) + activesupport (5.0.1) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (~> 0.7) + minitest (~> 5.1) + tzinfo (~> 1.1) + addressable (2.5.0) + public_suffix (~> 2.0, >= 2.0.2) + aws-sdk (2.7.0) + aws-sdk-resources (= 2.7.0) + aws-sdk-core (2.7.0) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-resources (2.7.0) + aws-sdk-core (= 2.7.0) + aws-sigv4 (1.0.0) + bson (1.12.5) + bzip2-ffi (1.0.0) + ffi (~> 1.0) + concurrent-ruby (1.0.4) + cool.io (1.4.5) + elasticsearch (1.0.18) + elasticsearch-api (= 1.0.18) + elasticsearch-transport (= 1.0.18) + elasticsearch-api (1.0.18) multi_json - elasticsearch-transport (1.0.17) + elasticsearch-transport (1.0.18) faraday multi_json - excon (0.49.0) - faraday (0.9.2) + excon (0.54.0) + faraday (0.11.0) multipart-post (>= 1.2, < 3) - fluent-plugin-elasticsearch (1.5.0) - elasticsearch + ffi (1.9.17) + fluent-mixin-config-placeholders (0.4.0) + fluentd + uuidtools (>= 2.1.5) + fluent-mixin-plaintextformatter (0.2.6) + fluentd + ltsv + fluent-plugin-elasticsearch (1.9.2) + elasticsearch (< 1.1) excon fluentd (>= 0.10.43) - fluent-plugin-record-reformer (0.8.1) + fluent-plugin-kafka (0.5.0) + fluentd (>= 0.10.58, < 2) + ltsv + ruby-kafka (= 0.3.16.beta2) + fluent-plugin-kinesis (1.1.2) + aws-sdk (~> 2) + concurrent-ruby (~> 1) + fluentd (>= 0.10.58, < 2) + os (>= 0.9.6) + protobuf (>= 3.5.5) + fluent-plugin-mongo (0.7.16) + fluentd (>= 0.10.58, < 2) + mongo (~> 1.9) + fluent-plugin-record-reformer (0.8.2) + fluentd + fluent-plugin-rewrite-tag-filter (1.5.5) fluentd - fluentd (0.14.0) - cool.io (>= 1.4.3, < 2.0.0) + fluent-plugin-s3 (0.8.0) + aws-sdk (>= 2.3.22, < 3) + fluentd (>= 0.10.58, < 2) + fluent-plugin-scribe (0.10.14) + fluentd + thrift (~> 0.8.0) + fluent-plugin-webhdfs (0.5.2) + bzip2-ffi + fluent-mixin-config-placeholders (>= 0.3.0) + fluent-mixin-plaintextformatter (>= 0.2.1) + fluentd (>= 0.10.59) + webhdfs (>= 0.6.0) + fluentd (0.14.11) + cool.io (~> 1.4.5) http_parser.rb (>= 0.5.1, < 0.7.0) - json (>= 1.4.3) - msgpack (>= 0.7.0) - serverengine (>= 1.6.4) + msgpack (>= 0.7.0, < 2.0.0) + serverengine (>= 2.0.4, < 3.0.0) sigdump (~> 0.2.2) - strptime (>= 0.1.7) - tzinfo (>= 1.0.0) - tzinfo-data (>= 1.0.0) + strptime (~> 0.1.7) + tzinfo (~> 1.0) + tzinfo-data (~> 1.0) yajl-ruby (~> 1.0) http_parser.rb (0.6.0) - json (1.8.3) - msgpack (0.7.6) + i18n (0.7.0) + jmespath (1.3.1) + ltsv (0.1.0) + middleware (0.1.0) + minitest (5.10.1) + mongo (1.12.5) + bson (= 1.12.5) + msgpack (1.0.2) multi_json (1.12.1) multipart-post (2.0.0) - serverengine (1.6.4) + os (0.9.6) + protobuf (3.6.12) + activesupport (>= 3.2) + middleware + thor + thread_safe + public_suffix (2.0.5) + ruby-kafka (0.3.16.beta2) + serverengine (2.0.4) sigdump (~> 0.2.2) sigdump (0.2.4) - strptime (0.1.8) + strptime (0.1.9) + thor (0.19.4) thread_safe (0.3.5) + thrift (0.8.0) tzinfo (1.2.2) thread_safe (~> 0.1) - tzinfo-data (1.2016.4) + tzinfo-data (1.2016.10) tzinfo (>= 1.0.0) - yajl-ruby (1.2.1) + uuidtools (2.1.5) + webhdfs (0.8.0) + addressable + yajl-ruby (1.3.0) PLATFORMS ruby DEPENDENCIES fluent-plugin-elasticsearch + fluent-plugin-kafka + fluent-plugin-kinesis + fluent-plugin-mongo fluent-plugin-record-reformer + fluent-plugin-rewrite-tag-filter + fluent-plugin-s3 + fluent-plugin-scribe + fluent-plugin-webhdfs fluentd BUNDLED WITH - 1.11.2 + 1.12.5 diff --git a/pkgs/tools/misc/fluentd/gemset.nix b/pkgs/tools/misc/fluentd/gemset.nix index e6b03fadfd36..1c508e7b58e2 100644 --- a/pkgs/tools/misc/fluentd/gemset.nix +++ b/pkgs/tools/misc/fluentd/gemset.nix @@ -1,75 +1,227 @@ { + activesupport = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "08bnl0nr9csjgkgz6xf8dyg7rccinmfrmn235z3bfaz8ihz15d1d"; + type = "gem"; + }; + version = "5.0.1"; + }; + addressable = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1j5r0anj8m4qlf2psnldip4b8ha2bsscv11lpdgnfh4nnchzjnxw"; + type = "gem"; + }; + version = "2.5.0"; + }; + aws-sdk = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "19s7ialas1yrc54g50yfa37z7m8dq4gqbf8dvlfg8qmpdijjxy3l"; + type = "gem"; + }; + version = "2.7.0"; + }; + aws-sdk-core = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0a9sgff43s3zhpcmisk1bp6vvlpawa617svfhz84xwa6lmik9sp4"; + type = "gem"; + }; + version = "2.7.0"; + }; + aws-sdk-resources = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1b5z25n4bgzwkzmzx2q6ik2y74jinyphmrh38lnrn9im6pmmvy3w"; + type = "gem"; + }; + version = "2.7.0"; + }; + aws-sigv4 = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0cnrfxbaxn86qrxhfidg10f89ka1hddihakdhcvnri0dljaw7dsz"; + type = "gem"; + }; + version = "1.0.0"; + }; + bson = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "12zcsfr72hr0w1qyxv1iz587nzganpclvimyx5y02gg1hij8hz6b"; + type = "gem"; + }; + version = "1.12.5"; + }; + bzip2-ffi = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1y5jlcz1vb0v3rbmsbbrarfglcmzdhr5jhlfc5wjnhz2zpybsz3y"; + type = "gem"; + }; + version = "1.0.0"; + }; + concurrent-ruby = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0p7ji1h1l407kci9w4b4yspzd58ssmlx7p91npx55kw08836dlpb"; + type = "gem"; + }; + version = "1.0.4"; + }; "cool.io" = { source = { remotes = ["https://rubygems.org"]; - sha256 = "0ycc8qdvpba8bf6da8nsna34md86mk527j4qizxh059vqm3521sb"; + sha256 = "1x5fkyjdjwk68sg7fwxhx2k3hzxkkm6frnd2yix7brxdh06fp0k1"; type = "gem"; }; - version = "1.4.4"; + version = "1.4.5"; }; elasticsearch = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1g7vax396l68w5mrrfbsaly39zkc4rrvljz9717mxyn82m5f66w5"; + sha256 = "1wdy17i56b4m7akp7yavnr8vhfhyz720waphmixq05dj21b11hl0"; type = "gem"; }; - version = "1.0.17"; + version = "1.0.18"; }; elasticsearch-api = { source = { remotes = ["https://rubygems.org"]; - sha256 = "08bb63raz381fmspijwjc4ksvrrgavmwrymjms1b9mg4qkic87jx"; + sha256 = "1v6nb3ajz5rack3p4b4nz37hs0zb9x738h2ms8cc4plp6wqh1w5s"; type = "gem"; }; - version = "1.0.17"; + version = "1.0.18"; }; elasticsearch-transport = { source = { remotes = ["https://rubygems.org"]; - sha256 = "07r798g3lnzr3zabk2ks2j5jnxdga23bc8wrr7mcqzn8q0yv82bz"; + sha256 = "0smfrz8nq49hgf67y5ayxa9i4rmmi0q4m51l0h499ykq4cvcwv6i"; type = "gem"; }; - version = "1.0.17"; + version = "1.0.18"; }; excon = { source = { remotes = ["https://rubygems.org"]; - sha256 = "0jmdgc4lhlbxccpg79a32vn3qngqipcaaq8bxa0ivfw5mvz0zc0z"; + sha256 = "0j4b6s90v84r4wrhbg4rzjfjg9sfisq50fjd3hh9p6yrkm86wbd3"; type = "gem"; }; - version = "0.49.0"; + version = "0.54.0"; }; faraday = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1kplqkpn2s2yl3lxdf6h7sfldqvkbkpxwwxhyk7mdhjplb5faqh6"; + sha256 = "18p1csdivgwmshfw3mb698a3bn0yrykg30khk5qxjf6n168g91jr"; + type = "gem"; + }; + version = "0.11.0"; + }; + ffi = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "07hnyr47mndsjfanzh348wm3fxjx9nx68mdb3cpsdvfqrxnz97s7"; + type = "gem"; + }; + version = "1.9.17"; + }; + fluent-mixin-config-placeholders = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "14b4lqy91jgpky6g7h0vyfy2rr1qavmjzzgnmhwajfzxgw9y2jvi"; + type = "gem"; + }; + version = "0.4.0"; + }; + fluent-mixin-plaintextformatter = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0gliangfr07060ya9sawkyfx2vz7vdygys65f83czawhckvvm75n"; type = "gem"; }; - version = "0.9.2"; + version = "0.2.6"; }; fluent-plugin-elasticsearch = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1kgv62s51y9x98qk0b6wrg4a73jfbhw50vg5z36hr0bh9rh2rq4y"; + sha256 = "0q0v8jxpwrkh1z5qh0chwrssz93nldka4jwfn32hlqhnmb99q8i1"; + type = "gem"; + }; + version = "1.9.2"; + }; + fluent-plugin-kafka = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0sd025xsl1cnjs11wasg0di2k02rx9ifaj49n28ak363df6vsqgf"; + type = "gem"; + }; + version = "0.5.0"; + }; + fluent-plugin-kinesis = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "119ngswi9q0p5hh5ldan9pzrgd1lfsbkr5f56hy1k4gfss4kmq27"; + type = "gem"; + }; + version = "1.1.2"; + }; + fluent-plugin-mongo = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1x7n8cknqh956yx3c9hv2g535x4kcixmnxw3fvcspjbqprrd1s91"; type = "gem"; }; - version = "1.5.0"; + version = "0.7.16"; }; fluent-plugin-record-reformer = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1ca09msvcdgrjv0xdjxh0nhxx8crp3h9nz5qw90c75s5hss2ws9b"; + sha256 = "1q2pws1mqp6pkb00ix6wjkxklckqb4wcbp79lpyk0b644bk9hqzb"; + type = "gem"; + }; + version = "0.8.2"; + }; + fluent-plugin-rewrite-tag-filter = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1avxvvmfm7bl7fpa2p73295kydh1nbsgdvsr7bsyrb77z1s1m86z"; type = "gem"; }; - version = "0.8.1"; + version = "1.5.5"; + }; + fluent-plugin-s3 = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0nxvk5n76pw4r37lv8vfl1cd0yjxnlj5wlwyk8f1lvp9ma5zlzmg"; + type = "gem"; + }; + version = "0.8.0"; + }; + fluent-plugin-scribe = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "00m19w7p22adq0yx1h7h2h4ckw9kh5j458a8lawgmbazw2dz0zxi"; + type = "gem"; + }; + version = "0.10.14"; + }; + fluent-plugin-webhdfs = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0kb9cgrgvh61pqqzv2csnibmp2jwh4hyjyvrh2npkk59k3jp54ad"; + type = "gem"; + }; + version = "0.5.2"; }; fluentd = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1v6c8g6fv9s710lrl0jy9ihbb8af37gvw3klk7csr5whp1mhwb8f"; + sha256 = "0w1bg3nrn6gwhyp8xlpbs9rcajkddnvw6jhn7kvzydp70g2aydhz"; type = "gem"; }; - version = "0.14.0"; + version = "0.14.11"; }; "http_parser.rb" = { source = { @@ -78,21 +230,61 @@ }; version = "0.6.0"; }; - json = { + i18n = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1i5z1ykl8zhszsxcs8mzl8d0dxgs3ylz8qlzrw74jb0gplkx6758"; + type = "gem"; + }; + version = "0.7.0"; + }; + jmespath = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "07w8ipjg59qavijq59hl82zs74jf3jsp7vxl9q3a2d0wpv5akz3y"; + type = "gem"; + }; + version = "1.3.1"; + }; + ltsv = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1alfq3g0mih4w86736ybnzqmknphm2z95c9q0wl765i4lrmxng11"; + type = "gem"; + }; + version = "0.1.0"; + }; + middleware = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0703nkf2v371wqr41c04x5qid7ww45cxqv3hnlg07if3b3xrm9xl"; + type = "gem"; + }; + version = "0.1.0"; + }; + minitest = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1yk2m8sp0p5m1niawa3ncg157a4i0594cg7z91rzjxv963rzrwab"; + type = "gem"; + }; + version = "5.10.1"; + }; + mongo = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1nsby6ry8l9xg3yw4adlhk2pnc7i0h0rznvcss4vk3v74qg0k8lc"; + sha256 = "0658pn2hbyfvbnpp3wdh3irin0wpikm6y2qbhnx07w54jbkmgh5p"; type = "gem"; }; - version = "1.8.3"; + version = "1.12.5"; }; msgpack = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1fn2riiaygiyvmr0glgm1vx995np3jb2hjf5i0j78vncd2wbwdw5"; + sha256 = "1fb2my91j08plsbbry5kilsrh7slmzgbbf6f55zy6xk28p9036lg"; type = "gem"; }; - version = "0.7.6"; + version = "1.0.2"; }; multi_json = { source = { @@ -109,13 +301,45 @@ }; version = "2.0.0"; }; + os = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1llv8w3g2jwggdxr5a5cjkrnbbfnvai3vxacxxc0fy84xmz3hymz"; + type = "gem"; + }; + version = "0.9.6"; + }; + protobuf = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0cvkfp7574dr6wqpgafl3pg9niqfri3dh2fxb2f8qaapcgfgcaq6"; + type = "gem"; + }; + version = "3.6.12"; + }; + public_suffix = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "040jf98jpp6w140ghkhw2hvc1qx41zvywx5gj7r2ylr1148qnj7q"; + type = "gem"; + }; + version = "2.0.5"; + }; + ruby-kafka = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "112avql9nf0hq07zvh47fyx7na721bj4zfpf43ip471l6k2ksrf5"; + type = "gem"; + }; + version = "0.3.16.beta2"; + }; serverengine = { source = { remotes = ["https://rubygems.org"]; - sha256 = "16sy6yissv8h2vla5ba4msqzsjy0cm0x8q2llssx3kl3bwysrbrp"; + sha256 = "0f08kbiqg9yp5fxdw5blsrnq383a9g4n830g1ypppb7ddv61sbmi"; type = "gem"; }; - version = "1.6.4"; + version = "2.0.4"; }; sigdump = { source = { @@ -128,10 +352,18 @@ strptime = { source = { remotes = ["https://rubygems.org"]; - sha256 = "0lkadizgdls9ya4sbf3bg5i1z6g2kxfw1r5ja0wkc9711zxjilx2"; + sha256 = "1avbl1fj4y5qx9ywkxpcjjxxpjj6h7r1dqlnddhk5wqg6ypq8lsb"; + type = "gem"; + }; + version = "0.1.9"; + }; + thor = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "01n5dv9kql60m6a00zc0r66jvaxx98qhdny3klyj0p3w34pad2ns"; type = "gem"; }; - version = "0.1.8"; + version = "0.19.4"; }; thread_safe = { source = { @@ -140,6 +372,14 @@ }; version = "0.3.5"; }; + thrift = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0aj23ndh5n5yqcvp4c12y7vl5wvxpl66zncf6n6ax2zvb6ig44cv"; + type = "gem"; + }; + version = "0.8.0"; + }; tzinfo = { dependencies = ["thread_safe"]; source = { @@ -151,16 +391,33 @@ tzinfo-data = { source = { remotes = ["https://rubygems.org"]; - sha256 = "1bxfljd5i7g89s7jc5l4a3ddykfsvvp0gm02805r1q77ahn1gp33"; + sha256 = "01nr50alfm1fyzlcbzvfbpnsq37yb3h676f9n3z0iyp4s4766psf"; + type = "gem"; + }; + version = "1.2016.10"; + }; + uuidtools = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0zjvq1jrrnzj69ylmz1xcr30skf9ymmvjmdwbvscncd7zkr8av5g"; + type = "gem"; + }; + version = "2.1.5"; + }; + webhdfs = { + source = { + remotes = ["https://rubygems.org"]; + sha256 = "0gs6xb9dz9bp5xc38yplfy48jcgj7jrj0zg0vgi7ydkxnkzkhbf2"; type = "gem"; }; - version = "1.2016.4"; + version = "0.8.0"; }; yajl-ruby = { source = { - sha256 = "0zvvb7i1bl98k3zkdrnx9vasq0rp2cyy5n7p9804dqs4fz9xh9vf"; + remotes = ["https://rubygems.org"]; + sha256 = "0sah2lpvpsh555dcnhgcqylinjj5544md9dh1a0a13da0qv1p57i"; type = "gem"; }; - version = "1.2.1"; + version = "1.3.0"; }; } \ No newline at end of file -- cgit 1.4.1