osquery: 3.2.9 -> 3.3.2

Although there's already a windows-only 3.4[1], the latest release for
Unix systems is 3.3.2[2] with the following changes:

* SELinux event recording[3]
* SMART drive information[4]
* ELF parsing to monitor compiler-level protections for binaries[5]

The diff became rather big in the end due to the following changes:

* The SMART information feature uses a custom fork of smartmontools to
  retrieve SMART data programatically[6].

* Replaced the old, bloated `misc.patch` by a simpler patch that only
  fixes two (actual) issues in the CMake build, unnecessary link targets
  and an invalid compiler flag.

* I tried to clean the derivation's code up a little bit by
  simplifying the `preConfigure` hook and adding only one build-input
  per line (that makes merging of several changes to the derivation
  easier as well).

[1] https://github.com/facebook/osquery/releases/tag/3.4.0
[2] https://github.com/facebook/osquery/releases/tag/3.3.2
[3] https://github.com/facebook/osquery/pull/4224
[4] https://github.com/facebook/osquery/pull/4133
[5] https://github.com/facebook/osquery/pull/4708
[6] https://github.com/allanliu/smartmontools
This commit is contained in:
Maximilian Bosch 2019-05-26 23:29:42 +02:00
parent d98d4b35ae
commit e4b49457af
No known key found for this signature in database
GPG key ID: 091DBF4D1FC46B8E
3 changed files with 140 additions and 238 deletions

View file

@ -0,0 +1,41 @@
From 511400b8ec0794588df88e2bb1c45cc6a4a51b1e Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Sun, 26 May 2019 23:10:02 +0200
Subject: [PATCH] Fix CMake configuration for Nix
* -fno-limit-debug-info: CLang specific, patch away to support both
CLang and GCC.
* Dropped unnecessary `lvm-internal` and `daemonclient` references.
---
CMakeLists.txt | 1 -
osquery/tables/CMakeLists.txt | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index f545d6c9..cfe68ddb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -328,7 +328,6 @@ if (POSIX)
# To be safe, only include them when building 'release' outputs.
add_compile_options(
"$<$<CONFIG:RELEASE>:-g>"
- "$<$<CONFIG:RELEASE>:-fno-limit-debug-info>"
"$<$<CONFIG:RELEASE>:-fPIE>"
"$<$<CONFIG:RELEASE>:-fpie>"
"$<$<CONFIG:RELEASE>:-fPIC>"
diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt
index 470bdf1d..caa6ab6f 100644
--- a/osquery/tables/CMakeLists.txt
+++ b/osquery/tables/CMakeLists.txt
@@ -68,7 +68,7 @@ if(LINUX)
set(TABLE_PLATFORM "linux")
ADD_OSQUERY_LINK_ADDITIONAL("libresolv.so")
- ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app lvm-internal daemonclient")
+ ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app")
ADD_OSQUERY_LINK_ADDITIONAL("gcrypt gpg-error")
ADD_OSQUERY_LINK_ADDITIONAL("blkid")
ADD_OSQUERY_LINK_ADDITIONAL("ip4tc")
--
2.19.2

View file

@ -1,58 +1,20 @@
{ stdenv, lib, fetchFromGitHub, pkgconfig, cmake, pythonPackages
{ stdenv, lib, fetchFromGitHub, pkgconfig, cmake, python
, udev, audit, aws-sdk-cpp, cryptsetup, lvm2, libgcrypt, libarchive
, libgpgerror, libuuid, iptables, dpkg, lzma, bzip2, rpm
, beecrypt, augeas, libxml2, sleuthkit, yara, lldpd, google-gflags
, thrift, boost, rocksdb_lite, glog, gbenchmark, snappy
, openssl, file, doxygen
, gtest, sqlite, fpm, zstd, rdkafka, rapidjson, fetchgit, fetchurl
, gtest, sqlite, fpm, zstd, rdkafka, rapidjson, fetchgit, fetchurl, libelfin
, smartmontools, which, git, cscope, ctags, ssdeep
}:
let
thirdparty = fetchFromGitHub {
owner = "osquery";
repo = "third-party";
rev = "32e01462fbea75d3b1904693f937dfd62eaced15";
sha256 = "0va24gmgk43a1lyjs63q9qrhvpv8gmqjzpjr5595vhr16idv8wyf";
};
in
stdenv.mkDerivation rec {
name = "osquery-${version}";
version = "3.2.9";
# this is what `osquery --help` will show as the version.
OSQUERY_BUILD_VERSION = version;
OSQUERY_PLATFORM = "NixOS;";
src = fetchFromGitHub {
owner = "facebook";
repo = "osquery";
rev = version;
sha256 = "1fac0yj1701469qhbsp38ab2fmavm3jw6x278bf78yvxdi99ivai";
};
patches = [ ./misc.patch ];
nativeBuildInputs = [
pkgconfig cmake pythonPackages.python pythonPackages.jinja2 doxygen fpm
];
NIX_LDFLAGS = [
"-lcrypto"
];
buildInputs = let
gflags' = google-gflags.overrideAttrs (old: {
cmakeFlags = stdenv.lib.filter (f: (builtins.match ".*STATIC.*" f) == null) old.cmakeFlags;
});
overrides = {
# use older `lvm2` source for osquery, the 2.03 sourcetree
# will break osquery due to the lacking header `lvm2app.h`.
#
# https://github.com/NixOS/nixpkgs/pull/51756#issuecomment-446035295
lvm2' = lvm2.overrideAttrs (old: rec {
lvm2 = lvm2.overrideAttrs (old: rec {
name = "lvm2-${version}";
version = "2.02.183";
src = fetchgit {
@ -62,8 +24,25 @@ stdenv.mkDerivation rec {
};
});
# use smartmontools fork to programatically retrieve SMART information.
# https://github.com/facebook/osquery/pull/4133
smartmontools = smartmontools.overrideAttrs (old: rec {
name = "smartmontools-${version}";
version = "0.3.1";
src = fetchFromGitHub {
owner = "allanliu";
repo = "smartmontools";
rev = "v${version}";
sha256 = "1i72fk2ranrky02h7nh9l3va4kjzj0lx1gr477zkxd44wf3w0pjf";
};
# Apple build fix doesn't apply here and isn't needed as we
# only support `osquery` on Linux.
patches = [];
});
# dpkg 1.19.2 dropped api in `<dpkg/dpkg-db.h>` which breaks compilation.
dpkg' = dpkg.overrideAttrs (old: rec {
dpkg = dpkg.overrideAttrs (old: rec {
name = "dpkg-${version}";
version = "1.19.0.5";
src = fetchurl {
@ -71,33 +50,92 @@ stdenv.mkDerivation rec {
sha256 = "1dc5kp3fqy1k66fly6jfxkkg7w6d0jy8szddpfyc2xvzga94d041";
};
});
in [
udev audit
# filter out static linking configuration to avoid that the library will
# be linked both statically and dynamically.
gflags = google-gflags.overrideAttrs (old: {
cmakeFlags = stdenv.lib.filter (f: (builtins.match ".*STATIC.*" f) == null) old.cmakeFlags;
});
};
in
stdenv.mkDerivation rec {
pname = "osquery";
version = "3.3.2";
# this is what `osquery --help` will show as the version.
OSQUERY_BUILD_VERSION = version;
OSQUERY_PLATFORM = "NixOS;";
src = fetchFromGitHub {
owner = "facebook";
repo = pname;
rev = version;
sha256 = "0nrwmzmbziacs3y0nljyc73bibr3w68myjpfwkicg9zgkq4qihij";
};
patches = [ ./0001-Fix-CMake-configuration-for-Nix.patch ];
NIX_CFLAGS_COMPILE = [
"-I${libxml2.dev}/include/libxml2"
];
nativeBuildInputs = [ python which git cscope ctags cmake pkgconfig doxygen fpm ]
++ (with python.pkgs; [ jinja2 ]);
buildInputs = [
udev
audit
(aws-sdk-cpp.override {
apis = [ "firehose" "kinesis" "sts" "ec2" ];
customMemoryManagement = false;
})
lvm2' libgcrypt libarchive libgpgerror libuuid iptables dpkg'
lzma bzip2 rpm beecrypt augeas libxml2 sleuthkit
yara lldpd gflags' thrift boost
glog gbenchmark snappy openssl
file cryptsetup
gtest sqlite zstd rdkafka rapidjson rocksdb_lite
overrides.lvm2
libgcrypt
libarchive
libgpgerror
libuuid
iptables
overrides.dpkg
lzma
bzip2
rpm
beecrypt
augeas
libxml2
sleuthkit
yara
lldpd
overrides.gflags
thrift
boost
glog
gbenchmark
snappy
openssl
file
cryptsetup
gtest
zstd
rdkafka
rapidjson
rocksdb_lite
libelfin
ssdeep
overrides.smartmontools
];
cmakeFlags = [ "-DSKIP_TESTS=1" ];
preConfigure = ''
export NIX_CFLAGS_COMPILE="-I${libxml2.dev}/include/libxml2 $NIX_CFLAGS_COMPILE"
cp -r ${fetchFromGitHub {
owner = "osquery";
repo = "third-party";
rev = "32e01462fbea75d3b1904693f937dfd62eaced15";
sha256 = "0va24gmgk43a1lyjs63q9qrhvpv8gmqjzpjr5595vhr16idv8wyf";
}}/* third-party
cmakeFlagsArray+=(
-DCMAKE_LIBRARY_PATH=${cryptsetup}/lib
-DCMAKE_VERBOSE_MAKEFILE=OFF
)
cp -r ${thirdparty}/* third-party
chmod +w -R third-party
rm -r third-party/{googletest,sqlite3}
'';
meta = with lib; {

View file

@ -1,177 +0,0 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0242fb71..5007aace 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -304,7 +304,6 @@ else()
# To be safe, only include them when building 'release' outputs.
add_compile_options(
-g
- -fno-limit-debug-info
-fPIE
-fpie
-fPIC
@@ -449,12 +448,6 @@ elseif(CLANG AND DEPS AND NOT FREEBSD)
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -flto=thin")
endif()
-if(NOT IS_DIRECTORY "${CMAKE_SOURCE_DIR}/third-party/sqlite3")
- WARNING_LOG("Cannot find git submodule third-party/sqlite3 directory")
- WARNING_LOG("Please run: make deps or git submodule update --init")
- message(FATAL_ERROR "No sqlite3 directory")
-endif()
-
# Make sure deps were built before compiling (else show warning).
execute_process(
COMMAND "${CMAKE_SOURCE_DIR}/tools/provision.sh" check "${CMAKE_BINARY_DIR}"
@@ -521,6 +514,8 @@ endif()
if(APPLE)
LOG_PLATFORM("OS X")
+elseif(OSQUERY_BUILD_PLATFORM STREQUAL "nixos")
+ LOG_PLATFORM("NixOS")
elseif(OSQUERY_BUILD_PLATFORM STREQUAL "debian")
LOG_PLATFORM("Debian")
elseif(OSQUERY_BUILD_PLATFORM STREQUAL "ubuntu")
@@ -570,7 +565,6 @@ if(POSIX AND DEPS)
endif()
endif()
-include_directories("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
include_directories("${CMAKE_SOURCE_DIR}/include")
include_directories("${CMAKE_SOURCE_DIR}")
@@ -661,18 +655,6 @@ if(FREEBSD OR "${HAVE_TR1_TUPLE}" STREQUAL "")
add_definitions(-DGTEST_USE_OWN_TR1_TUPLE=0)
endif()
-set(GTEST_FLAGS
- "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googletest/include"
- "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googlemock/include"
-)
-join("${GTEST_FLAGS}" " " GTEST_FLAGS)
-
-if(NOT SKIP_TESTS)
- add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/googletest")
-endif()
-
-add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
-
if(NOT FREEBSD)
add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/linenoise-ng")
endif()
diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt
index e3842962..f405503d 100644
--- a/osquery/CMakeLists.txt
+++ b/osquery/CMakeLists.txt
@@ -33,8 +33,6 @@ if(CLANG AND POSIX)
-Wextra
-pedantic
-Wuseless-cast
- -Wno-c99-extensions
- -Wno-zero-length-array
-Wno-unused-parameter
-Wno-gnu-case-range
-Weffc++
@@ -63,14 +61,7 @@ endif()
# Construct a set of all object files, starting with third-party and all
# of the osquery core objects (sources from ADD_CORE_LIBRARY macros).
-if(FREEBSD)
- set(OSQUERY_OBJECTS $<TARGET_OBJECTS:osquery_sqlite>)
-else()
- set(OSQUERY_OBJECTS
- $<TARGET_OBJECTS:osquery_sqlite>
- $<TARGET_OBJECTS:linenoise-ng>
- )
-endif()
+set(OSQUERY_OBJECTS $<TARGET_OBJECTS:linenoise-ng>)
# Add subdirectories
add_subdirectory(config)
@@ -151,10 +142,11 @@ if(APPLE OR LINUX)
ADD_OSQUERY_LINK_ADDITIONAL("rocksdb_lite")
elseif(FREEBSD)
ADD_OSQUERY_LINK_CORE("icuuc")
- ADD_OSQUERY_LINK_CORE("linenoise")
ADD_OSQUERY_LINK_ADDITIONAL("rocksdb-lite")
endif()
+ADD_OSQUERY_LINK_CORE("sqlite3")
+
if(POSIX)
ADD_OSQUERY_LINK_CORE("boost_system")
ADD_OSQUERY_LINK_CORE("boost_filesystem")
@@ -172,10 +164,10 @@ endif()
ADD_OSQUERY_LINK_CORE("glog${WO_KEY}")
if(POSIX)
- # Hashing methods in core use libcrypto.
- ADD_OSQUERY_LINK_CORE("crypto")
-
+ ADD_OSQUERY_LINK_ADDITIONAL("benchmark")
+ ADD_OSQUERY_LINK_ADDITIONAL("snappy")
ADD_OSQUERY_LINK_ADDITIONAL("ssl")
+ ADD_OSQUERY_LINK_ADDITIONAL("crypto")
ADD_OSQUERY_LINK_ADDITIONAL("libpthread")
ADD_OSQUERY_LINK_ADDITIONAL("magic")
endif()
diff --git a/osquery/extensions/CMakeLists.txt b/osquery/extensions/CMakeLists.txt
index 52f3bf80..066ed1c0 100644
--- a/osquery/extensions/CMakeLists.txt
+++ b/osquery/extensions/CMakeLists.txt
@@ -60,12 +60,6 @@ else()
)
endif()
-if(NOT WINDOWS)
- add_compile_options(
- -Wno-macro-redefined
- )
-endif()
-
ADD_OSQUERY_LIBRARY(TRUE osquery_extensions
${THRIFT_GENERATED_FILES}
${THRIFT_IMPL_FILE}
diff --git a/osquery/logger/CMakeLists.txt b/osquery/logger/CMakeLists.txt
index ab91bd24..d8364991 100644
--- a/osquery/logger/CMakeLists.txt
+++ b/osquery/logger/CMakeLists.txt
@@ -55,9 +55,9 @@ if(NOT SKIP_KAFKA AND NOT WINDOWS AND NOT FREEBSD)
ADD_OSQUERY_LINK_ADDITIONAL("rdkafka")
- set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
- "logger/plugins/tests/kafka_producer_tests.cpp"
- )
+ #set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
+ # "logger/plugins/tests/kafka_producer_tests.cpp"
+ #)
ADD_OSQUERY_TEST_ADDITIONAL(${OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS})
endif()
diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt
index 3ecbb711..af7220d3 100644
--- a/osquery/tables/CMakeLists.txt
+++ b/osquery/tables/CMakeLists.txt
@@ -68,7 +68,7 @@ if(LINUX)
set(TABLE_PLATFORM "linux")
ADD_OSQUERY_LINK_ADDITIONAL("libresolv.so")
- ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app lvm-internal daemonclient")
+ ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app")
ADD_OSQUERY_LINK_ADDITIONAL("gcrypt gpg-error")
ADD_OSQUERY_LINK_ADDITIONAL("blkid")
ADD_OSQUERY_LINK_ADDITIONAL("ip4tc")
diff --git a/specs/windows/services.table b/specs/windows/services.table
index 4ac24ee9..657d8b99 100644
--- a/specs/windows/services.table
+++ b/specs/windows/services.table
@@ -12,7 +12,7 @@ schema([
Column("path", TEXT, "Path to Service Executable"),
Column("module_path", TEXT, "Path to ServiceDll"),
Column("description", TEXT, "Service Description"),
- Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\UserName. If the account belongs to the built-in domain, the name can be of the form .\UserName."),
+ Column("user_account", TEXT, "The name of the account that the service process will be logged on as when it runs. This name can be of the form Domain\\UserName. If the account belongs to the built-in domain, the name can be of the form .\\UserName."),
])
implementation("system/windows/services@genServices")
examples([