Merge pull request #27347 from cstrahan/osquery-new

osquery: init at 2.5.2

7 files changed, 391 insertions, 0 deletions

diff --git a/pkgs/development/libraries/linenoise-ng/default.nix b/pkgs/development/libraries/linenoise-ng/default.nix
new file mode 100644
index 000000000000..b3333d58242e
--- /dev/null
+++ b/pkgs/development/libraries/linenoise-ng/default.nix
@@ -0,0 +1,23 @@
+{ stdenv, lib, fetchFromGitHub, cmake }:
+
+stdenv.mkDerivation rec {
+  name = "linenoise-ng-${version}";
+  version = "1.0.1";
+
+  src = fetchFromGitHub {
+    owner = "arangodb";
+    repo = "linenoise-ng";
+    rev = "v${version}";
+    sha256 = "176iz0kj0p8d8i3jqps4z8xkxwl3f1986q88i9xg5fvqgpzsxp20";
+  };
+
+  nativeBuildInputs = [ cmake ];
+
+  meta = {
+    homepage = "https://github.com/arangodb/linenoise-ng";
+    description = "A small, portable GNU readline replacement for Linux, Windows and MacOS which is capable of handling UTF-8 characters";
+    maintainers = with stdenv.lib.maintainers; [ cstrahan ];
+    platforms = stdenv.lib.platforms.all;
+    license = stdenv.lib.licenses.bsd3;
+  };
+}
diff --git a/pkgs/development/libraries/rocksdb/default.nix b/pkgs/development/libraries/rocksdb/default.nix
index 25aeb3a6da1b..a791c8994856 100644
--- a/pkgs/development/libraries/rocksdb/default.nix
+++ b/pkgs/development/libraries/rocksdb/default.nix
@@ -6,6 +6,8 @@
 
 # Malloc implementation
 , jemalloc ? null, gperftools ? null
+
+, enableLite ? false
 }:
 
 let
@@ -35,6 +37,9 @@ stdenv.mkDerivation rec {
   CMAKE_CXX_FLAGS = "-std=gnu++11";
   JEMALLOC_LIB = stdenv.lib.optionalString (malloc == jemalloc) "-ljemalloc";
 
+  ${if enableLite then "LIBNAME" else null} = "librocksdb_lite";
+  ${if enableLite then "CXXFLAGS" else null} = "-DROCKSDB_LITE=1";
+
   buildFlags = [
     "DEBUG_LEVEL=0"
     "shared_lib"
diff --git a/pkgs/tools/package-management/apt/default.nix b/pkgs/tools/package-management/apt/default.nix
new file mode 100644
index 000000000000..de1c24050761
--- /dev/null
+++ b/pkgs/tools/package-management/apt/default.nix
@@ -0,0 +1,65 @@
+{ stdenv, lib, fetchzip, pkgconfig, cmake, perl, curl, gtest, lzma, bzip2 , lz4
+, db, dpkg, libxslt, docbook_xsl, docbook_xml_dtd_45
+
+# used when WITH_DOC=ON
+, w3m
+, Po4a
+, doxygen
+
+# used when WITH_NLS=ON
+, gettext
+
+# opts
+, withDocs ? true
+, withNLS ? true
+}:
+
+stdenv.mkDerivation rec {
+  name = "apt-${version}";
+
+  version = "1.4.6";
+
+  src = fetchzip {
+    url = "https://launchpad.net/ubuntu/+archive/primary/+files/apt_${version}.tar.xz";
+    sha256 = "0ahwhmscrmnpvl1r732wg93dzkhv8c1sph2yrqgsrhr73c1616ix";
+  };
+
+  buildInputs = [
+    pkgconfig cmake perl curl gtest lzma bzip2 lz4 db dpkg libxslt.bin
+  ] ++ lib.optionals withDocs [
+    doxygen Po4a w3m
+  ] ++ lib.optionals withNLS [
+    gettext
+  ];
+
+  preConfigure = ''
+    export PERL5LIB="$PERL5LIB''${PERL5LIB:+:}${Po4a}/lib/perl5";
+
+    cmakeFlagsArray+=(
+      -DBERKELEY_DB_INCLUDE_DIRS="${db}"/include
+      -DDOCBOOK_XSL="${docbook_xsl}"/share/xml/docbook-xsl
+      -DROOT_GROUP=root
+      -DWITH_DOC=${if withDocs then "ON" else "OFF"}
+      -DUSE_NLS=${if withNLS then "ON" else "OFF"}
+    )
+
+    for f in doc/*; do
+      if [[ -f "$f" ]]; then
+        substituteInPlace "$f" \
+          --replace \
+            "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd" \
+            "${docbook_xml_dtd_45}/xml/dtd/docbook/docbookx.dtd"
+      fi
+    done
+  '';
+
+  enableParallelBuilding = true;
+
+  meta = with lib; {
+    description = "";
+    homepage = "https://launchpad.net/ubuntu/+source/apt";
+    license = licenses.gpl2Plus;
+    platforms = platforms.linux;
+    maintainers = with maintainers; [ cstrahan ];
+  };
+}
diff --git a/pkgs/tools/system/osquery/default.nix b/pkgs/tools/system/osquery/default.nix
new file mode 100644
index 000000000000..7924054b7203
--- /dev/null
+++ b/pkgs/tools/system/osquery/default.nix
@@ -0,0 +1,79 @@
+{ stdenv, lib, fetchFromGitHub, pkgconfig, cmake, pythonPackages
+, udev, audit, aws-sdk-cpp, cryptsetup, lvm2, libgcrypt, libarchive
+, libgpgerror, libuuid, iptables, apt, dpkg, lzma, lz4, bzip2, rpm
+, beecrypt, augeas, libxml2, sleuthkit, yara, lldpd, google-gflags
+, thrift, boost, rocksdb_lite, cpp-netlib, glog, gbenchmark, snappy
+, openssl, linenoise-ng, file, doxygen, devicemapper
+, gtest, sqlite
+}:
+
+let
+  thirdparty = fetchFromGitHub {
+    owner = "osquery";
+    repo = "third-party";
+    rev = "6919841175b2c9cb2dee8986e0cfe49191ecb868";
+    sha256 = "1kjxrky586jd1b2z1vs9cm7x1dxw51cizpys9kddiarapc2ih65j";
+  };
+
+in
+
+stdenv.mkDerivation rec {
+  name = "osquery-${version}";
+  version = "2.5.2";
+
+  # this is what `osquery --help` will show as the version.
+  OSQUERY_BUILD_VERSION = version;
+
+  src = fetchFromGitHub {
+    owner = "facebook";
+    repo = "osquery";
+    rev = version;
+    sha256 = "16isplk66qpvhrf041l0lxb4z6k7wwd1sg7kpsw2q6kivkxpnk3z";
+  };
+
+  patches = [ ./misc.patch ] ++ lib.optional stdenv.isLinux ./platform-nixos.patch;
+
+  nativeBuildInputs = [
+    pkgconfig cmake pythonPackages.python pythonPackages.jinja2
+  ];
+
+  buildInputs = [
+    udev audit
+
+    (aws-sdk-cpp.override {
+      apis = [ "firehose" "kinesis" "sts" ];
+      customMemoryManagement = false;
+    })
+
+    lvm2 libgcrypt libarchive libgpgerror libuuid iptables.dev apt dpkg
+    lzma lz4 bzip2 rpm beecrypt augeas libxml2 sleuthkit
+    yara lldpd google-gflags thrift boost
+    cpp-netlib glog gbenchmark snappy openssl linenoise-ng
+    file doxygen devicemapper cryptsetup
+    gtest sqlite
+
+    # need to be consistent about the malloc implementation
+    (rocksdb_lite.override { jemalloc = null; gperftools = null; })
+  ];
+
+  preConfigure = ''
+    export NIX_CFLAGS_COMPILE="-I${libxml2.dev}/include/libxml2 $NIX_CFLAGS_COMPILE"
+
+    cmakeFlagsArray+=(
+      -DCMAKE_LIBRARY_PATH=${cryptsetup}/lib
+      -DCMAKE_VERBOSE_MAKEFILE=OFF
+    )
+
+    cp -r ${thirdparty}/* third-party
+    chmod +w -R third-party
+    rm -r third-party/{googletest,sqlite3}
+  '';
+
+  meta = with lib; {
+    description = "SQL powered operating system instrumentation, monitoring, and analytics";
+    homepage = "https://osquery.io/";
+    license = licenses.bsd3;
+    platforms = platforms.linux;
+    maintainers = with maintainers; [ cstrahan ];
+  };
+}
diff --git a/pkgs/tools/system/osquery/misc.patch b/pkgs/tools/system/osquery/misc.patch
new file mode 100644
index 000000000000..1a0ef267f0df
--- /dev/null
+++ b/pkgs/tools/system/osquery/misc.patch
@@ -0,0 +1,187 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index a976a46d..408ac308 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -125,14 +125,13 @@ else()
+     set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libc++")
+   else()
+     set(LINUX TRUE)
+-    set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libstdc++")
++    set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14")
+   endif()
+   set(POSIX TRUE)
+ endif()
+ 
+ if(POSIX)
+   add_compile_options(
+-    -Qunused-arguments
+     -Wstrict-aliasing
+     -Wno-missing-field-initializers
+     -Wno-unused-local-typedef
+@@ -154,7 +153,6 @@ if(POSIX)
+   )
+   if(NOT FREEBSD)
+     add_compile_options(
+-      -Werror=shadow
+       -fvisibility=hidden
+       -fvisibility-inlines-hidden
+     )
+@@ -372,12 +370,6 @@ elseif(NOT FREEBSD)
+   endif()
+ endif()
+ 
+-if(NOT IS_DIRECTORY "${CMAKE_SOURCE_DIR}/third-party/sqlite3")
+-  WARNING_LOG("Cannot find git submodule third-party/sqlite3 directory")
+-  WARNING_LOG("Please run: make deps or git submodule update --init")
+-  message(FATAL_ERROR "No sqlite3 directory")
+-endif()
+-
+ # Make sure deps were built before compiling (else show warning).
+ execute_process(
+   COMMAND "${CMAKE_SOURCE_DIR}/tools/provision.sh" check "${CMAKE_BINARY_DIR}"
+@@ -439,6 +431,8 @@ endif()
+ 
+ if(APPLE)
+   LOG_PLATFORM("OS X")
++elseif(OSQUERY_BUILD_PLATFORM STREQUAL "nixos")
++  LOG_PLATFORM("NixOS")
+ elseif(OSQUERY_BUILD_PLATFORM STREQUAL "debian")
+   LOG_PLATFORM("Debian")
+ elseif(OSQUERY_BUILD_PLATFORM STREQUAL "ubuntu")
+@@ -477,7 +471,6 @@ if(POSIX)
+   include_directories("${BUILD_DEPS}/include/openssl")
+ endif()
+ 
+-include_directories("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
+ include_directories("${CMAKE_SOURCE_DIR}/include")
+ include_directories("${CMAKE_SOURCE_DIR}")
+ 
+@@ -559,21 +552,10 @@ else()
+   set(GTEST_FLAGS "-DGTEST_USE_OWN_TR1_TUPLE=0")
+ endif()
+ 
+-set(GTEST_FLAGS
+-  ${GTEST_FLAGS}
+-  "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googletest/include"
+-  "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googlemock/include"
+-)
+-join("${GTEST_FLAGS}" " " GTEST_FLAGS)
+-
+ set(BUILD_GTEST TRUE)
+ 
+-add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/googletest")
+-
+ include(Thrift)
+ 
+-add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
+-
+ add_subdirectory(osquery)
+ add_subdirectory(tools/tests)
+ 
+diff --git a/include/osquery/core.h b/include/osquery/core.h
+index b597edee..b0628037 100644
+--- a/include/osquery/core.h
++++ b/include/osquery/core.h
+@@ -15,8 +15,9 @@
+ #include <string>
+ #include <vector>
+ 
+-#if defined(__APPLE__) || defined(__FreeBSD__)
++#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__)
+ #include <boost/thread/shared_mutex.hpp>
++#include <boost/thread/recursive_mutex.hpp>
+ #else
+ #include <shared_mutex>
+ #endif
+@@ -188,7 +189,7 @@ inline bool isPlatform(PlatformType a, const PlatformType& t = kPlatformType) {
+   return (static_cast<int>(t) & static_cast<int>(a)) != 0;
+ }
+ 
+-#if defined(__APPLE__) || defined(__FreeBSD__)
++#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__)
+ #define MUTEX_IMPL boost
+ #else
+ #define MUTEX_IMPL std
+@@ -204,10 +205,10 @@ using WriteLock = MUTEX_IMPL::unique_lock<Mutex>;
+ using ReadLock = MUTEX_IMPL::shared_lock<Mutex>;
+ 
+ /// Helper alias for defining recursive mutexes.
+-using RecursiveMutex = std::recursive_mutex;
++using RecursiveMutex = MUTEX_IMPL::recursive_mutex;
+ 
+ /// Helper alias for write locking a recursive mutex.
+-using RecursiveLock = std::lock_guard<std::recursive_mutex>;
++using RecursiveLock = MUTEX_IMPL::lock_guard<MUTEX_IMPL::recursive_mutex>;
+ }
+ 
+ /**
+diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt
+index 77913d31..671b20d4 100644
+--- a/osquery/CMakeLists.txt
++++ b/osquery/CMakeLists.txt
+@@ -57,7 +57,7 @@ endif()
+ 
+ # Construct a set of all object files, starting with third-party and all
+ # of the osquery core objects (sources from ADD_CORE_LIBRARY macros).
+-set(OSQUERY_OBJECTS $<TARGET_OBJECTS:osquery_sqlite>)
++set(OSQUERY_OBJECTS "")
+ 
+ # Add subdirectories
+ add_subdirectory(config)
+@@ -138,6 +138,8 @@ elseif(FREEBSD)
+   ADD_OSQUERY_LINK_ADDITIONAL("rocksdb-lite")
+ endif()
+ 
++ADD_OSQUERY_LINK_CORE("sqlite3")
++
+ if(POSIX)
+   ADD_OSQUERY_LINK_CORE("boost_system")
+   ADD_OSQUERY_LINK_CORE("boost_filesystem")
+@@ -157,6 +159,7 @@ ADD_OSQUERY_LINK_ADDITIONAL("cppnetlib-client-connections${WO_KEY}")
+ ADD_OSQUERY_LINK_CORE("glog${WO_KEY}")
+ 
+ if(POSIX)
++  ADD_OSQUERY_LINK_ADDITIONAL("benchmark")
+   ADD_OSQUERY_LINK_ADDITIONAL("snappy")
+   ADD_OSQUERY_LINK_ADDITIONAL("ssl")
+   ADD_OSQUERY_LINK_ADDITIONAL("crypto")
+@@ -336,13 +339,6 @@ if(NOT OSQUERY_BUILD_SDK_ONLY)
+ 
+   install(DIRECTORY "${CMAKE_SOURCE_DIR}/packs/"
+     DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/packs" COMPONENT main)
+-  if(APPLE)
+-    install(FILES "${CMAKE_SOURCE_DIR}/tools/deployment/com.facebook.osqueryd.plist"
+-      DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/" COMPONENT main)
+-  else()
+-    install(PROGRAMS "${CMAKE_SOURCE_DIR}/tools/deployment/osqueryd.initd"
+-      DESTINATION "/etc/init.d/" RENAME "osqueryd" COMPONENT main)
+-  endif()
+ endif()
+ 
+ if(NOT SKIP_TESTS)
+diff --git a/osquery/tables/system/linux/tests/md_tables_tests.cpp b/osquery/tables/system/linux/tests/md_tables_tests.cpp
+index 126be362..119d361d 100644
+--- a/osquery/tables/system/linux/tests/md_tables_tests.cpp
++++ b/osquery/tables/system/linux/tests/md_tables_tests.cpp
+@@ -72,7 +72,7 @@ void GetDrivesForArrayTestHarness(std::string arrayName,
+   EXPECT_CALL(md, getArrayInfo(arrayDevPath, _))
+       .WillOnce(DoAll(SetArgReferee<1>(arrayInfo), Return(true)));
+ 
+-  Sequence::Sequence s1;
++  Sequence s1;
+   for (int i = 0; i < MD_SB_DISKS; i++) {
+     mdu_disk_info_t diskInfo;
+     diskInfo.number = i;
+diff --git a/specs/windows/services.table b/specs/windows/services.table
+index 4ac24ee9..657d8b99 100644
+--- a/specs/windows/services.table
++++ b/specs/windows/services.table
+@@ -12,7 +12,7 @@ schema([
+     Column("path", TEXT, "Path to Service Executable"),
+     Column("module_path", TEXT, "Path to ServiceDll"),
+     Column("description", TEXT, "Service Description"),
+-    Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\UserName. If the account belongs to the built-in domain, the name can be of the form .\UserName."),
++    Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\\UserName. If the account belongs to the built-in domain, the name can be of the form .\\UserName."),
+ ])
+ implementation("system/windows/services@genServices")
+ examples([
diff --git a/pkgs/tools/system/osquery/platform-nixos.patch b/pkgs/tools/system/osquery/platform-nixos.patch
new file mode 100644
index 000000000000..7e1afcb640b3
--- /dev/null
+++ b/pkgs/tools/system/osquery/platform-nixos.patch
@@ -0,0 +1,22 @@
+diff --git a/tools/get_platform.py b/tools/get_platform.py
+index 3dd34516..f53ca83a 100644
+--- a/tools/get_platform.py
++++ b/tools/get_platform.py
+@@ -26,6 +26,8 @@ DEBIAN_VERSION = "/etc/debian_version"
+ GENTOO_RELEASE = "/etc/gentoo-release"
+ 
+ def _platform():
++    return ("nixos", "nixos")
++
+     osType, _, _, _, _, _ = platform.uname()
+ 
+     if osType == "Windows":
+@@ -75,6 +77,8 @@ def _platform():
+         return (None, osType.lower())
+ 
+ def _distro(osType):
++    return "unknown_version"
++
+     def getRedhatDistroVersion(pattern):
+         with open(SYSTEM_RELEASE, "r") as fd:
+             contents = fd.read()
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 4b9f9d7ae6ce..0333d7ef857c 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -531,6 +531,10 @@ with pkgs;
 
   apg = callPackage ../tools/security/apg { };
 
+  apt = callPackage ../tools/package-management/apt {
+    inherit (perlPackages) Po4a;
+  };
+
   autorevision = callPackage ../tools/misc/autorevision { };
 
   bcachefs-tools = callPackage ../tools/filesystems/bcachefs-tools { };
@@ -9349,6 +9353,8 @@ with pkgs;
 
   linenoise = callPackage ../development/libraries/linenoise { };
 
+  linenoise-ng = callPackage ../development/libraries/linenoise-ng { };
+
   lirc = callPackage ../development/libraries/lirc { };
 
   liquid-dsp = callPackage ../development/libraries/liquid-dsp { };
@@ -10088,6 +10094,8 @@ with pkgs;
 
   rocksdb = callPackage ../development/libraries/rocksdb { };
 
+  rocksdb_lite = rocksdb.override { enableLite = true; };
+
   rote = callPackage ../development/libraries/rote { };
 
   ronn = callPackage ../development/tools/ronn { };
@@ -15511,6 +15519,8 @@ with pkgs;
 
   osmo = callPackage ../applications/office/osmo { };
 
+  osquery = callPackage ../tools/system/osquery { };
+
   palemoon = callPackage ../applications/networking/browsers/palemoon {
     # https://forum.palemoon.org/viewtopic.php?f=57&t=15296#p111146
     stdenv = overrideCC stdenv gcc49;