osquery: 2.5.2 -> 3.2.2

The package was originally broken as reported in #38940 and
facebook/osquery#4257. The latest version (3.x) contains several
important fixes for GCC 7, so now we can compile without a much less
complicated patches.

The following changes were needed to fix the derivation:

* Upgrade `osquery/third-party` to the latest rev to be compliant with
  osquery 3.

* Keep using an override for the AWS SDK (for a lower closure size and
  less compile time), but make the `ec2` API available.

* Added the dependencies `fpm`, `zstd`, `rdkafka`, `rapidjson` to the
  build. `linenoise-ng` is obsolete as it's directly bundled with
  `osquery/third-party`.

* Fixed the linking issue with `gflags` as recommended in the mailing
  list: https://groups.google.com/d/msg/nix-devel/l1blj-mWxtI/J3CwPATBCAAJ

* Dropped the obsolete dependencies `cpp-netlib`, `lz4`, `apt` and
  `devicemapper` (thanks @Infinisil).

* Override `OSQUERY_PLATFORM` to provide `nixos:version`
  for sandbox and non-NixOS based builds. The `platform-nixos.patch`
  file is now obsolete (thanks @flokli).

The patch was rebased against the 3.x branch of `osquery` and contains
mostly old changes. Additionally several testing targets were skipped as
they broke the build.

The functionality has been testing using the following command:

```
mkdir /tmp/osq.log/
./result/bin/osqueryd --pidfile /tmp/osq.pid \
  --database_path /tmp/test.db --logger_path /tmp/osq.log
```

With the daemon running the database can be queried easily using
`./result/bin/osqueryi`.

Fixes ticket #38940
See ticket #36453

Further reference can be gathered from the affected Hydra logs for
the master branch: https://hydra.nixos.org/job/nixos/trunk-combined/nixpkgs.osquery.x86_64-linux
This commit is contained in:
Maximilian Bosch 2018-04-24 07:13:49 +02:00
parent e109784f84
commit 3cd2707e37
No known key found for this signature in database
GPG Key ID: 091DBF4D1FC46B8E
3 changed files with 111 additions and 154 deletions

View File

@ -1,59 +1,62 @@
{ stdenv, lib, fetchFromGitHub, pkgconfig, cmake, pythonPackages
, udev, audit, aws-sdk-cpp, cryptsetup, lvm2, libgcrypt, libarchive
, libgpgerror, libuuid, iptables, apt, dpkg, lzma, lz4, bzip2, rpm
, libgpgerror, libuuid, iptables, dpkg, lzma, bzip2, rpm
, beecrypt, augeas, libxml2, sleuthkit, yara, lldpd, google-gflags
, thrift, boost, rocksdb_lite, cpp-netlib, glog, gbenchmark, snappy
, openssl, linenoise-ng, file, doxygen, devicemapper
, gtest, sqlite
, thrift, boost, rocksdb_lite, glog, gbenchmark, snappy
, openssl, file, doxygen
, gtest, sqlite, fpm, zstd, rdkafka, rapidjson, path
}:
let
thirdparty = fetchFromGitHub {
owner = "osquery";
repo = "third-party";
rev = "6919841175b2c9cb2dee8986e0cfe49191ecb868";
sha256 = "1kjxrky586jd1b2z1vs9cm7x1dxw51cizpys9kddiarapc2ih65j";
rev = "4ef099c31a1165c5e7e3a699f9e4b3eb68c3c3d9";
sha256 = "1vm0prw4dix0m51vkw9z0vwfd8698gqjw499q8h604hs1rvn6132";
};
in
stdenv.mkDerivation rec {
name = "osquery-${version}";
version = "2.5.2";
version = "3.2.2";
# this is what `osquery --help` will show as the version.
OSQUERY_BUILD_VERSION = version;
OSQUERY_PLATFORM = "nixos;${builtins.readFile "${toString path}/.version"}";
src = fetchFromGitHub {
owner = "facebook";
repo = "osquery";
rev = version;
sha256 = "16isplk66qpvhrf041l0lxb4z6k7wwd1sg7kpsw2q6kivkxpnk3z";
sha256 = "0qwj4cy6m25sqwb0irqfqinipx50l4imnz1gqxx147vzfwb52jlq";
};
patches = [ ./misc.patch ] ++ lib.optional stdenv.isLinux ./platform-nixos.patch;
patches = [ ./misc.patch ];
nativeBuildInputs = [
pkgconfig cmake pythonPackages.python pythonPackages.jinja2
pkgconfig cmake pythonPackages.python pythonPackages.jinja2 doxygen fpm
];
buildInputs = [
buildInputs = let
gflags' = google-gflags.overrideAttrs (old: {
cmakeFlags = stdenv.lib.filter (f: isNull (builtins.match ".*STATIC.*" f)) old.cmakeFlags;
});
in [
udev audit
(aws-sdk-cpp.override {
apis = [ "firehose" "kinesis" "sts" ];
apis = [ "firehose" "kinesis" "sts" "ec2" ];
customMemoryManagement = false;
})
lvm2 libgcrypt libarchive libgpgerror libuuid iptables.dev apt dpkg
lzma lz4 bzip2 rpm beecrypt augeas libxml2 sleuthkit
yara lldpd google-gflags thrift boost
cpp-netlib glog gbenchmark snappy openssl linenoise-ng
file doxygen devicemapper cryptsetup
gtest sqlite
# need to be consistent about the malloc implementation
(rocksdb_lite.override { jemalloc = null; gperftools = null; })
lvm2 libgcrypt libarchive libgpgerror libuuid iptables dpkg
lzma bzip2 rpm beecrypt augeas libxml2 sleuthkit
yara lldpd gflags' thrift boost
glog gbenchmark snappy openssl
file cryptsetup
gtest sqlite zstd rdkafka rapidjson rocksdb_lite
];
preConfigure = ''
@ -74,7 +77,6 @@ stdenv.mkDerivation rec {
homepage = https://osquery.io/;
license = licenses.bsd3;
platforms = platforms.linux;
maintainers = with maintainers; [ cstrahan ];
broken = true; # 2018-04-11
maintainers = with maintainers; [ cstrahan ma27 ];
};
}

View File

@ -1,33 +1,9 @@
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a976a46d..408ac308 100644
index b8eb5a16..319d81dc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -125,14 +125,13 @@ else()
set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libc++")
else()
set(LINUX TRUE)
- set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14 -stdlib=libstdc++")
+ set(CXX_COMPILE_FLAGS "${CXX_COMPILE_FLAGS} -std=c++14")
endif()
set(POSIX TRUE)
endif()
if(POSIX)
add_compile_options(
- -Qunused-arguments
-Wstrict-aliasing
-Wno-missing-field-initializers
-Wno-unused-local-typedef
@@ -154,7 +153,6 @@ if(POSIX)
)
if(NOT FREEBSD)
add_compile_options(
- -Werror=shadow
-fvisibility=hidden
-fvisibility-inlines-hidden
)
@@ -372,12 +370,6 @@ elseif(NOT FREEBSD)
endif()
@@ -447,12 +447,6 @@ elseif(CLANG AND DEPS AND NOT FREEBSD)
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -flto=thin")
endif()
-if(NOT IS_DIRECTORY "${CMAKE_SOURCE_DIR}/third-party/sqlite3")
@ -39,7 +15,7 @@ index a976a46d..408ac308 100644
# Make sure deps were built before compiling (else show warning).
execute_process(
COMMAND "${CMAKE_SOURCE_DIR}/tools/provision.sh" check "${CMAKE_BINARY_DIR}"
@@ -439,6 +431,8 @@ endif()
@@ -518,6 +512,8 @@ endif()
if(APPLE)
LOG_PLATFORM("OS X")
@ -48,87 +24,67 @@ index a976a46d..408ac308 100644
elseif(OSQUERY_BUILD_PLATFORM STREQUAL "debian")
LOG_PLATFORM("Debian")
elseif(OSQUERY_BUILD_PLATFORM STREQUAL "ubuntu")
@@ -477,7 +471,6 @@ if(POSIX)
include_directories("${BUILD_DEPS}/include/openssl")
@@ -567,7 +563,6 @@ if(POSIX AND DEPS)
endif()
endif()
-include_directories("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
include_directories("${CMAKE_SOURCE_DIR}/include")
include_directories("${CMAKE_SOURCE_DIR}")
@@ -559,21 +552,10 @@ else()
set(GTEST_FLAGS "-DGTEST_USE_OWN_TR1_TUPLE=0")
@@ -655,18 +650,6 @@ if(FREEBSD OR "${HAVE_TR1_TUPLE}" STREQUAL "")
add_definitions(-DGTEST_USE_OWN_TR1_TUPLE=0)
endif()
-set(GTEST_FLAGS
- ${GTEST_FLAGS}
- "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googletest/include"
- "-I${CMAKE_SOURCE_DIR}/third-party/googletest/googlemock/include"
-)
-join("${GTEST_FLAGS}" " " GTEST_FLAGS)
-
set(BUILD_GTEST TRUE)
-add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/googletest")
-if(NOT SKIP_TESTS)
- add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/googletest")
-endif()
-
include(Thrift)
-add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/sqlite3")
-
add_subdirectory(osquery)
add_subdirectory(tools/tests)
diff --git a/include/osquery/core.h b/include/osquery/core.h
index b597edee..b0628037 100644
--- a/include/osquery/core.h
+++ b/include/osquery/core.h
@@ -15,8 +15,9 @@
#include <string>
#include <vector>
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__)
#include <boost/thread/shared_mutex.hpp>
+#include <boost/thread/recursive_mutex.hpp>
#else
#include <shared_mutex>
#endif
@@ -188,7 +189,7 @@ inline bool isPlatform(PlatformType a, const PlatformType& t = kPlatformType) {
return (static_cast<int>(t) & static_cast<int>(a)) != 0;
}
-#if defined(__APPLE__) || defined(__FreeBSD__)
+#if defined(__APPLE__) || defined(__FreeBSD__) || defined(__linux__)
#define MUTEX_IMPL boost
#else
#define MUTEX_IMPL std
@@ -204,10 +205,10 @@ using WriteLock = MUTEX_IMPL::unique_lock<Mutex>;
using ReadLock = MUTEX_IMPL::shared_lock<Mutex>;
/// Helper alias for defining recursive mutexes.
-using RecursiveMutex = std::recursive_mutex;
+using RecursiveMutex = MUTEX_IMPL::recursive_mutex;
/// Helper alias for write locking a recursive mutex.
-using RecursiveLock = std::lock_guard<std::recursive_mutex>;
+using RecursiveLock = MUTEX_IMPL::lock_guard<MUTEX_IMPL::recursive_mutex>;
}
/**
if(NOT FREEBSD)
add_subdirectory("${CMAKE_SOURCE_DIR}/third-party/linenoise-ng")
endif()
diff --git a/osquery/CMakeLists.txt b/osquery/CMakeLists.txt
index 77913d31..671b20d4 100644
index 1c8201ee..a89e6821 100644
--- a/osquery/CMakeLists.txt
+++ b/osquery/CMakeLists.txt
@@ -57,7 +57,7 @@ endif()
@@ -35,8 +35,6 @@ if(CLANG AND POSIX)
-Wextra
-pedantic
-Wuseless-cast
- -Wno-c99-extensions
- -Wno-zero-length-array
-Wno-unused-parameter
-Wno-gnu-case-range
-Weffc++
@@ -61,14 +59,7 @@ endif()
# Construct a set of all object files, starting with third-party and all
# of the osquery core objects (sources from ADD_CORE_LIBRARY macros).
-set(OSQUERY_OBJECTS $<TARGET_OBJECTS:osquery_sqlite>)
+set(OSQUERY_OBJECTS "")
-if(FREEBSD)
- set(OSQUERY_OBJECTS $<TARGET_OBJECTS:osquery_sqlite>)
-else()
- set(OSQUERY_OBJECTS
- $<TARGET_OBJECTS:osquery_sqlite>
- $<TARGET_OBJECTS:linenoise-ng>
- )
-endif()
+set(OSQUERY_OBJECTS $<TARGET_OBJECTS:linenoise-ng>)
# Add subdirectories
add_subdirectory(config)
@@ -138,6 +138,8 @@ elseif(FREEBSD)
@@ -147,10 +138,11 @@ if(APPLE OR LINUX)
ADD_OSQUERY_LINK_ADDITIONAL("rocksdb_lite")
elseif(FREEBSD)
ADD_OSQUERY_LINK_CORE("icuuc")
- ADD_OSQUERY_LINK_CORE("linenoise")
ADD_OSQUERY_LINK_ADDITIONAL("rocksdb-lite")
endif()
@ -137,41 +93,62 @@ index 77913d31..671b20d4 100644
if(POSIX)
ADD_OSQUERY_LINK_CORE("boost_system")
ADD_OSQUERY_LINK_CORE("boost_filesystem")
@@ -157,6 +159,7 @@ ADD_OSQUERY_LINK_ADDITIONAL("cppnetlib-client-connections${WO_KEY}")
@@ -168,6 +160,8 @@ endif()
ADD_OSQUERY_LINK_CORE("glog${WO_KEY}")
if(POSIX)
+ ADD_OSQUERY_LINK_ADDITIONAL("benchmark")
ADD_OSQUERY_LINK_ADDITIONAL("snappy")
+ ADD_OSQUERY_LINK_ADDITIONAL("snappy")
ADD_OSQUERY_LINK_ADDITIONAL("ssl")
ADD_OSQUERY_LINK_ADDITIONAL("crypto")
@@ -336,13 +339,6 @@ if(NOT OSQUERY_BUILD_SDK_ONLY)
install(DIRECTORY "${CMAKE_SOURCE_DIR}/packs/"
DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/packs" COMPONENT main)
- if(APPLE)
- install(FILES "${CMAKE_SOURCE_DIR}/tools/deployment/com.facebook.osqueryd.plist"
- DESTINATION "${CMAKE_INSTALL_PREFIX}/share/osquery/" COMPONENT main)
- else()
- install(PROGRAMS "${CMAKE_SOURCE_DIR}/tools/deployment/osqueryd.initd"
- DESTINATION "/etc/init.d/" RENAME "osqueryd" COMPONENT main)
- endif()
ADD_OSQUERY_LINK_ADDITIONAL("libpthread")
diff --git a/osquery/extensions/CMakeLists.txt b/osquery/extensions/CMakeLists.txt
index 52f3bf80..066ed1c0 100644
--- a/osquery/extensions/CMakeLists.txt
+++ b/osquery/extensions/CMakeLists.txt
@@ -60,12 +60,6 @@ else()
)
endif()
if(NOT SKIP_TESTS)
diff --git a/osquery/tables/system/linux/tests/md_tables_tests.cpp b/osquery/tables/system/linux/tests/md_tables_tests.cpp
index 126be362..119d361d 100644
--- a/osquery/tables/system/linux/tests/md_tables_tests.cpp
+++ b/osquery/tables/system/linux/tests/md_tables_tests.cpp
@@ -72,7 +72,7 @@ void GetDrivesForArrayTestHarness(std::string arrayName,
EXPECT_CALL(md, getArrayInfo(arrayDevPath, _))
.WillOnce(DoAll(SetArgReferee<1>(arrayInfo), Return(true)));
-if(NOT WINDOWS)
- add_compile_options(
- -Wno-macro-redefined
- )
-endif()
-
ADD_OSQUERY_LIBRARY(TRUE osquery_extensions
${THRIFT_GENERATED_FILES}
${THRIFT_IMPL_FILE}
diff --git a/osquery/logger/CMakeLists.txt b/osquery/logger/CMakeLists.txt
index ab91bd24..d8364991 100644
--- a/osquery/logger/CMakeLists.txt
+++ b/osquery/logger/CMakeLists.txt
@@ -55,9 +55,9 @@ if(NOT SKIP_KAFKA AND NOT WINDOWS AND NOT FREEBSD)
- Sequence::Sequence s1;
+ Sequence s1;
for (int i = 0; i < MD_SB_DISKS; i++) {
mdu_disk_info_t diskInfo;
diskInfo.number = i;
ADD_OSQUERY_LINK_ADDITIONAL("rdkafka")
- set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
- "logger/plugins/tests/kafka_producer_tests.cpp"
- )
+ #set(OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS
+ # "logger/plugins/tests/kafka_producer_tests.cpp"
+ #)
ADD_OSQUERY_TEST_ADDITIONAL(${OSQUERY_LOGGER_KAFKA_PLUGINS_TESTS})
endif()
diff --git a/osquery/tables/CMakeLists.txt b/osquery/tables/CMakeLists.txt
index dd78084f..158758e1 100644
--- a/osquery/tables/CMakeLists.txt
+++ b/osquery/tables/CMakeLists.txt
@@ -68,7 +68,7 @@ if(LINUX)
set(TABLE_PLATFORM "linux")
ADD_OSQUERY_LINK_ADDITIONAL("libresolv.so")
- ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app lvm-internal daemonclient")
+ ADD_OSQUERY_LINK_ADDITIONAL("cryptsetup devmapper lvm2app")
ADD_OSQUERY_LINK_ADDITIONAL("gcrypt gpg-error")
ADD_OSQUERY_LINK_ADDITIONAL("blkid")
ADD_OSQUERY_LINK_ADDITIONAL("ip4tc")
diff --git a/specs/windows/services.table b/specs/windows/services.table
index 4ac24ee9..657d8b99 100644
--- a/specs/windows/services.table

View File

@ -1,22 +0,0 @@
diff --git a/tools/get_platform.py b/tools/get_platform.py
index 3dd34516..f53ca83a 100644
--- a/tools/get_platform.py
+++ b/tools/get_platform.py
@@ -26,6 +26,8 @@ DEBIAN_VERSION = "/etc/debian_version"
GENTOO_RELEASE = "/etc/gentoo-release"
def _platform():
+ return ("nixos", "nixos")
+
osType, _, _, _, _, _ = platform.uname()
if osType == "Windows":
@@ -75,6 +77,8 @@ def _platform():
return (None, osType.lower())
def _distro(osType):
+ return "unknown_version"
+
def getRedhatDistroVersion(pattern):
with open(SYSTEM_RELEASE, "r") as fd:
contents = fd.read()