osquery: 2.5.2 -> 3.2.2

The package was originally broken as reported in #38940 and
facebook/osquery#4257. The latest version (3.x) contains several
important fixes for GCC 7, so now we can compile without a much less
complicated patches.

The following changes were needed to fix the derivation:

* Upgrade `osquery/third-party` to the latest rev to be compliant with
  osquery 3.

* Keep using an override for the AWS SDK (for a lower closure size and
  less compile time), but make the `ec2` API available.

* Added the dependencies `fpm`, `zstd`, `rdkafka`, `rapidjson` to the
  build. `linenoise-ng` is obsolete as it's directly bundled with
  `osquery/third-party`.

* Fixed the linking issue with `gflags` as recommended in the mailing
  list: https://groups.google.com/d/msg/nix-devel/l1blj-mWxtI/J3CwPATBCAAJ

* Dropped the obsolete dependencies `cpp-netlib`, `lz4`, `apt` and
  `devicemapper` (thanks @Infinisil).

* Override `OSQUERY_PLATFORM` to provide `nixos:version`
  for sandbox and non-NixOS based builds. The `platform-nixos.patch`
  file is now obsolete (thanks @flokli).

The patch was rebased against the 3.x branch of `osquery` and contains
mostly old changes. Additionally several testing targets were skipped as
they broke the build.

The functionality has been testing using the following command:

```
mkdir /tmp/osq.log/
./result/bin/osqueryd --pidfile /tmp/osq.pid \
  --database_path /tmp/test.db --logger_path /tmp/osq.log
```

With the daemon running the database can be queried easily using
`./result/bin/osqueryi`.

Fixes ticket #38940
See ticket #36453

Further reference can be gathered from the affected Hydra logs for
the master branch: https://hydra.nixos.org/job/nixos/trunk-combined/nixpkgs.osquery.x86_64-linux

1 files changed, 87 insertions, 110 deletions

diff --git a/pkgs/tools/system/osquery/misc.patch b/pkgs/tools/system/osquery/misc.patch
index 1a0ef267f0df..acdbf6346c83 100644
--- a/pkgs/tools/system/osquery/misc.patch
+++ b/pkgs/tools/system/osquery/misc.patch
@@ -1,33 +1,9 @@
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index a976a46d..408ac308 100644
+index b8eb5a16..319d81dc 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
-@@ -125,14 +125,13 @@ else()
-     set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libc++")
-   else()
-     set(LINUX TRUE)
--    set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libstdc++")
-+    set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14")
-   endif()
-   set(POSIX TRUE)
- endif()
- 
- if(POSIX)
-   add_compile_options(
--    -Qunused-arguments
-     -Wstrict-aliasing
-     -Wno-missing-field-initializers
-     -Wno-unused-local-typedef
-@@ -154,7 +153,6 @@ if(POSIX)
-   )
-   if(NOT FREEBSD)
-     add_compile_options(
--      -Werror=shadow
-       -fvisibility=hidden
-       -fvisibility-inlines-hidden
-     )
-@@ -372,12 +370,6 @@ elseif(NOT FREEBSD)
-   endif()
+@@ -447,12 +447,6 @@ elseif(CLANG AND DEPS AND NOT FREEBSD)
+   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -flto=thin")
  endif()
  
 -if(NOT IS_DIRECTORY "${CMAKE_SOURCE_DIR}/third-party/sqlite3")
@@ -39,7 +15,7 @@ index a976a46d..408ac308 100644
  # Make sure deps were built before compiling (else show warning).
  execute_process(
    COMMAND "${CMAKE_SOURCE_DIR}/tools/provision.sh" check "${CMAKE_BINARY_DIR}"
-@@ -439,6 +431,8 @@ endif()
+@@ -518,6 +512,8 @@ endif()
  
  if(APPLE)
    LOG_PLATFORM("OS X")
@@ -48,87 +24,67 @@ index a976a46d..408ac308 100644
  elseif(OSQUERY_BUILD_PLATFORM STREQUAL "debian")
    LOG_PLATFORM("Debian")
  elseif(OSQUERY_BUILD_PLATFORM STREQUAL "ubuntu")
-@@ -477,7 +471,6 @@ if(POSIX)
-   include_directories("${BUILD_DEPS}/include/openssl")
+@@ -567,7 +563,6 @@ if(POSIX AND DEPS)
+   endif()
  endif()
  
 -include_directories("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
  include_directories("${CMAKE_SOURCE_DIR}/include")
  include_directories("${CMAKE_SOURCE_DIR}")
  
-@@ -559,21 +552,10 @@ else()
-   set(GTEST_FLAGS "-DGTEST_USE_OWN_TR1_TUPLE=0")
+@@ -655,18 +650,6 @@ if(FREEBSD OR "${HAVE_TR1_TUPLE}" STREQUAL "")
+   add_definitions(-DGTEST_USE_OWN_TR1_TUPLE=0)
  endif()
  
 -set(GTEST_FLAGS
--  ${GTEST_FLAGS}
 -  "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googletest/include"
 -  "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googlemock/include"
 -)
 -join("${GTEST_FLAGS}" " " GTEST_FLAGS)
 -
- set(BUILD_GTEST TRUE)
- 
--add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/googletest")
+-if(NOT SKIP_TESTS)
+-  add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/googletest")
+-endif()
 -
- include(Thrift)
- 
 -add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
 -
- add_subdirectory(osquery)
- add_subdirectory(tools/tests)
- 
-diff --git a/include/osquery/core.h b/include/osquery/core.h
-index b597edee..b0628037 100644
---- a/include/osquery/core.h
-+++ b/include/osquery/core.h
-@@ -15,8 +15,9 @@
- #include <string>
- #include <vector>
- 
--#if defined(__APPLE__) || defined(__FreeBSD__)
-+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__)
- #include <boost/thread/shared_mutex.hpp>
-+#include <boost/thread/recursive_mutex.hpp>
- #else
- #include <shared_mutex>
- #endif
-@@ -188,7 +189,7 @@ inline bool isPlatform(PlatformType a, const PlatformType& t = kPlatformType) {
-   return (static_cast<int>(t) & static_cast<int>(a)) != 0;
- }
- 
--#if defined(__APPLE__) || defined(__FreeBSD__)
-+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__)
- #define MUTEX_IMPL boost
- #else
- #define MUTEX_IMPL std
-@@ -204,10 +205,10 @@ using WriteLock = MUTEX_IMPL::unique_lock<Mutex>;
- using ReadLock = MUTEX_IMPL::shared_lock<Mutex>;
- 
- /// Helper alias for defining recursive mutexes.
--using RecursiveMutex = std::recursive_mutex;
-+using RecursiveMutex = MUTEX_IMPL::recursive_mutex;
- 
- /// Helper alias for write locking a recursive mutex.
--using RecursiveLock = std::lock_guard<std::recursive_mutex>;
-+using RecursiveLock = MUTEX_IMPL::lock_guard<MUTEX_IMPL::recursive_mutex>;
- }
- 
- /**
+ if(NOT FREEBSD)
+   add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/linenoise-ng")
+ endif()
 diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt
-index 77913d31..671b20d4 100644
+index 1c8201ee..a89e6821 100644
 --- a/osquery/CMakeLists.txt
 +++ b/osquery/CMakeLists.txt
-@@ -57,7 +57,7 @@ endif()
+@@ -35,8 +35,6 @@ if(CLANG AND POSIX)
+     -Wextra
+     -pedantic
+     -Wuseless-cast
+-    -Wno-c99-extensions
+-    -Wno-zero-length-array
+     -Wno-unused-parameter
+     -Wno-gnu-case-range
+     -Weffc++
+@@ -61,14 +59,7 @@ endif()
  
  # Construct a set of all object files, starting with third-party and all
  # of the osquery core objects (sources from ADD_CORE_LIBRARY macros).
--set(OSQUERY_OBJECTS $<TARGET_OBJECTS:osquery_sqlite>)
-+set(OSQUERY_OBJECTS "")
+-if(FREEBSD)
+-  set(OSQUERY_OBJECTS $<TARGET_OBJECTS:osquery_sqlite>)
+-else()
+-  set(OSQUERY_OBJECTS
+-    $<TARGET_OBJECTS:osquery_sqlite>
+-    $<TARGET_OBJECTS:linenoise-ng>
+-  )
+-endif()
++set(OSQUERY_OBJECTS $<TARGET_OBJECTS:linenoise-ng>)
  
  # Add subdirectories
  add_subdirectory(config)
-@@ -138,6 +138,8 @@ elseif(FREEBSD)
+@@ -147,10 +138,11 @@ if(APPLE OR LINUX)
+   ADD_OSQUERY_LINK_ADDITIONAL("rocksdb_lite")
+ elseif(FREEBSD)
+   ADD_OSQUERY_LINK_CORE("icuuc")
+-  ADD_OSQUERY_LINK_CORE("linenoise")
    ADD_OSQUERY_LINK_ADDITIONAL("rocksdb-lite")
  endif()
  
@@ -137,41 +93,62 @@ index 77913d31..671b20d4 100644
  if(POSIX)
    ADD_OSQUERY_LINK_CORE("boost_system")
    ADD_OSQUERY_LINK_CORE("boost_filesystem")
-@@ -157,6 +159,7 @@ ADD_OSQUERY_LINK_ADDITIONAL("cppnetlib-client-connections${WO_KEY}")
+@@ -168,6 +160,8 @@ endif()
  ADD_OSQUERY_LINK_CORE("glog${WO_KEY}")
  
  if(POSIX)
 +  ADD_OSQUERY_LINK_ADDITIONAL("benchmark")
-   ADD_OSQUERY_LINK_ADDITIONAL("snappy")
++  ADD_OSQUERY_LINK_ADDITIONAL("snappy")
    ADD_OSQUERY_LINK_ADDITIONAL("ssl")
    ADD_OSQUERY_LINK_ADDITIONAL("crypto")
-@@ -336,13 +339,6 @@ if(NOT OSQUERY_BUILD_SDK_ONLY)
- 
-   install(DIRECTORY "${CMAKE_SOURCE_DIR}/packs/"
-     DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/packs" COMPONENT main)
--  if(APPLE)
--    install(FILES "${CMAKE_SOURCE_DIR}/tools/deployment/com.facebook.osqueryd.plist"
--      DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/" COMPONENT main)
--  else()
--    install(PROGRAMS "${CMAKE_SOURCE_DIR}/tools/deployment/osqueryd.initd"
--      DESTINATION "/etc/init.d/" RENAME "osqueryd" COMPONENT main)
--  endif()
+   ADD_OSQUERY_LINK_ADDITIONAL("libpthread")
+diff --git a/osquery/extensions/CMakeLists.txt b/osquery/extensions/CMakeLists.txt
+index 52f3bf80..066ed1c0 100644
+--- a/osquery/extensions/CMakeLists.txt
++++ b/osquery/extensions/CMakeLists.txt
+@@ -60,12 +60,6 @@ else()
+   )
  endif()
  
- if(NOT SKIP_TESTS)
-diff --git a/osquery/tables/system/linux/tests/md_tables_tests.cpp b/osquery/tables/system/linux/tests/md_tables_tests.cpp
-index 126be362..119d361d 100644
---- a/osquery/tables/system/linux/tests/md_tables_tests.cpp
-+++ b/osquery/tables/system/linux/tests/md_tables_tests.cpp
-@@ -72,7 +72,7 @@ void GetDrivesForArrayTestHarness(std::string arrayName,
-   EXPECT_CALL(md, getArrayInfo(arrayDevPath, _))
-       .WillOnce(DoAll(SetArgReferee<1>(arrayInfo), Return(true)));
- 
--  Sequence::Sequence s1;
-+  Sequence s1;
-   for (int i = 0; i < MD_SB_DISKS; i++) {
-     mdu_disk_info_t diskInfo;
-     diskInfo.number = i;
+-if(NOT WINDOWS)
+-  add_compile_options(
+-    -Wno-macro-redefined
+-  )
+-endif()
+-
+ ADD_OSQUERY_LIBRARY(TRUE osquery_extensions
+   ${THRIFT_GENERATED_FILES}
+   ${THRIFT_IMPL_FILE}
+diff --git a/osquery/logger/CMakeLists.txt b/osquery/logger/CMakeLists.txt
+index ab91bd24..d8364991 100644
+--- a/osquery/logger/CMakeLists.txt
++++ b/osquery/logger/CMakeLists.txt
+@@ -55,9 +55,9 @@ if(NOT SKIP_KAFKA AND NOT WINDOWS AND NOT FREEBSD)
+ 
+   ADD_OSQUERY_LINK_ADDITIONAL("rdkafka")
+ 
+-  set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
+-    "logger/plugins/tests/kafka_producer_tests.cpp"
+-  )
++  #set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
++  #  "logger/plugins/tests/kafka_producer_tests.cpp"
++  #)
+ 
+   ADD_OSQUERY_TEST_ADDITIONAL(${OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS})
+ endif()
+diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt
+index dd78084f..158758e1 100644
+--- a/osquery/tables/CMakeLists.txt
++++ b/osquery/tables/CMakeLists.txt
+@@ -68,7 +68,7 @@ if(LINUX)
+   set(TABLE_PLATFORM "linux")
+ 
+   ADD_OSQUERY_LINK_ADDITIONAL("libresolv.so")
+-  ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app lvm-internal daemonclient")
++  ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app")
+   ADD_OSQUERY_LINK_ADDITIONAL("gcrypt gpg-error")
+   ADD_OSQUERY_LINK_ADDITIONAL("blkid")
+   ADD_OSQUERY_LINK_ADDITIONAL("ip4tc")
 diff --git a/specs/windows/services.table b/specs/windows/services.table
 index 4ac24ee9..657d8b99 100644
 --- a/specs/windows/services.table