foundationdb: init 6.1.6pre4898_26fbbbf, cmake build

This adds a new build of FoundationDB 6.1, using the new, much improved
with CMake build system with fewer patches and rough edges.

Signed-off-by: Austin Seipp <aseipp@pobox.com>
This commit is contained in:
Austin Seipp 2019-05-02 21:44:11 -05:00
parent a13a0c8de4
commit 3f1c6801a2
No known key found for this signature in database
GPG key ID: 25D2038DEB08021D
8 changed files with 328 additions and 4 deletions

View file

@ -0,0 +1,129 @@
# This builder is for FoundationDB CMake build system.
{ lib, fetchurl, fetchpatch, fetchFromGitHub
, cmake, ninja, boost, python3, openjdk, mono, libressl
, gccStdenv, llvmPackages
, useClang ? true
, ...
}:
let
stdenv = if useClang then llvmPackages.libcxxStdenv else gccStdenv;
tests = with builtins;
builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt);
makeFdb =
{ version
, branch
, sha256
, rev ? "refs/tags/${version}"
, officialRelease ? true
, patches ? []
}: stdenv.mkDerivation rec {
name = "foundationdb-${version}";
inherit version;
src = fetchFromGitHub {
owner = "apple";
repo = "foundationdb";
inherit rev sha256;
};
buildInputs = [ libressl boost ];
nativeBuildInputs = [ cmake ninja python3 openjdk mono ]
++ lib.optional useClang [ llvmPackages.lld ];
separateDebugInfo = true;
enableParallelBuilding = true;
dontFixCmake = true;
cmakeFlags =
[ "-DCMAKE_BUILD_TYPE=Release"
"-DLIBRESSL_USE_STATIC_LIBS=FALSE"
# CMake can't find these easily for some reason?
"-DLIBRESSL_INCLUDE_DIR=${libressl.dev}"
"-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so"
"-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so"
"-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so"
# LTO brings up overall build time, but results in much smaller
# binaries for all users and the cache.
#
# TODO FIXME: bugs :(
(lib.optionalString (!useClang) "-DUSE_LTO=ON")
# Gold helps alleviate the link time, especially when LTO is
# enabled. But even then, it still takes a majority of the time.
# Same with LLD when Clang is available.
(lib.optionalString useClang "-DUSE_LD=LLD")
(lib.optionalString (!useClang) "-DUSE_LD=GOLD")
]
++ lib.optional officialRelease [ "FDB_RELEASE=1" ];
inherit patches;
postPatch = ''
for x in bindings/c/CMakeLists.txt fdbserver/CMakeLists.txt fdbmonitor/CMakeLists.txt fdbbackup/CMakeLists.txt fdbcli/CMakeLists.txt; do
substituteInPlace $x --replace 'fdb_install' 'install'
done
'';
# the install phase for cmake is pretty wonky right now since it's not designed to
# coherently install packages as most linux distros expect -- it's designed to build
# packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to
# cmake upstream to fix this, and if we do, i think most of this can go away.
postInstall = ''
mv $out/sbin/fdbserver $out/bin/fdbserver
rm -rf \
$out/lib/systemd $out/Library $out/usr $out/sbin \
$out/var $out/log $out/etc
mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor
rm -rf $out/lib/foundationdb/
mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent
mkdir $out/include/foundationdb && \
mv $out/include/*.h $out/include/*.options $out/include/foundationdb
# move results into multi outputs
mkdir -p $dev $lib
mv $out/include $dev/include
mv $out/lib $lib/lib
# python bindings
# NB: use the original setup.py.in, so we can substitute VERSION correctly
cp ../LICENSE ./bindings/python
substitute ../bindings/python/setup.py.in ./bindings/python/setup.py \
--replace 'VERSION' "${version}"
rm -f ./bindings/python/setup.py.* ./bindings/python/CMakeLists.txt
rm -f ./bindings/python/fdb/*.pth # remove useless files
rm -f ./bindings/python/*.rst ./bindings/python/*.mk
cp -R ./bindings/python/ tmp-pythonsrc/
tar -zcf $pythonsrc --transform s/tmp-pythonsrc/python-foundationdb/ ./tmp-pythonsrc/
# java bindings
mkdir -p $lib/share/java
mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar
# include the tests
mkdir -p $out/share/test
(cd ../tests && for x in ${tests}; do
cp --parents $x $out/share/test
done)
'';
outputs = [ "out" "dev" "lib" "pythonsrc" ];
meta = with stdenv.lib; {
description = "Open source, distributed, transactional key-value store";
homepage = https://www.foundationdb.org;
license = licenses.asl20;
platforms = [ "x86_64-linux" ];
maintainers = with maintainers; [ thoughtpolice ];
};
};
in makeFdb

View file

@ -1,12 +1,16 @@
{ stdenv49
{ stdenv, stdenv49, gcc9Stdenv, llvmPackages_8
, lib, fetchurl, fetchpatch, fetchFromGitHub
, which, findutils, m4, gawk
, python, openjdk, mono, libressl
, cmake, ninja, which, findutils, m4, gawk
, python, python3, openjdk, mono, libressl, boost
}@args:
let
vsmakeBuild = import ./vsmake.nix args;
cmakeBuild = import ./cmake.nix (args // {
gccStdenv = gcc9Stdenv;
llvmPackages = llvmPackages_8;
});
python3-six-patch = fetchpatch {
name = "update-python-six.patch";
@ -21,6 +25,10 @@ let
};
in with builtins; {
# Older versions use the bespoke 'vsmake' build system
# ------------------------------------------------------
foundationdb51 = vsmakeBuild rec {
version = "5.1.7";
branch = "release-5.1";
@ -56,4 +64,21 @@ in with builtins; {
./patches/ldflags-6.0.patch
];
};
# 6.1 and later versions should always use CMake
# ------------------------------------------------------
foundationdb61 = cmakeBuild rec {
version = "6.1.6pre4898_${substring 0 7 rev}";
branch = "release-6.1";
rev = "26fbbbf798971b2b9ecb882a8af766fa36734f53";
sha256 = "1q1a1j8h0qlh67khcds0dg416myvjbp6gfm6s4sk8d60zfzny7wb";
officialRelease = false;
patches = [
./patches/clang-libcxx.patch
./patches/suppress-clang-warnings.patch
];
};
}

View file

@ -0,0 +1,52 @@
commit 7ed4745a092a203f92fc37ab5894e92117db0c94
Author: Austin Seipp <aseipp@pobox.com>
Date: Sat May 4 15:23:35 2019 -0500
flow: fix a build failure with Clang/libcxx on Linux
11bd7d7da introduced a hack on Linux to work around a missing symbol in
libstdc++'s _pic library on Ubuntu. Unfortunately, this causes the build
to fail when using Clang, as it doesn't believe this symbol is part of
its headers in c++11 mode.
Unfortunately there's no good way to distinguish libcxx from libstdc++
with the preprocessor, so we merely gate it by only checking for clang,
iff we are on Linux.
With this change, Clang 8.x can build FoundationDB on Linux using libcxx
as the standard C++ library.
Signed-off-by: Austin Seipp <aseipp@pobox.com>
diff --git a/flow/Platform.cpp b/flow/Platform.cpp
index 3d3f1ac0..9f21dfd4 100644
--- a/flow/Platform.cpp
+++ b/flow/Platform.cpp
@@ -2841,13 +2841,26 @@ void setupSlowTaskProfiler() {
#endif
}
-#ifdef __linux__
+#if defined(__linux__) && !defined(__clang__)
// There's no good place to put this, so it's here.
// Ubuntu's packaging of libstdc++_pic offers different symbols than libstdc++. Go figure.
// Notably, it's missing a definition of std::istream::ignore(long), which causes compilation errors
// in the bindings. Thus, we provide weak versions of their definitions, so that if the
// linked-against libstdc++ is missing their definitions, we'll be able to use the provided
// ignore(long, int) version.
+//
+// Note that this hack is DISABLED when we use Clang. It is only needed when we statically link
+// to the _pic libraries, but only official FDB Linux binaries are built this way using GCC. If we
+// don't use the _pic libraries, then this hack is entirely unneeded -- likely the case when using
+// Clang on Linux.
+//
+// Doing this allows us to use LLVM's libc++ with Clang on Linux -- otherwise, providing
+// a weak symbol definition for an internal (non-public) class member fails (due to that member
+// being non-existant on libc++.) See upstream GitHub issue #1533 for more information.
+//
+// TODO FIXME: Obliterate this when the official build environment is upgraded beyond Ubuntu 14.04.
+// (This problem should be fixed in later LTS releases.)
+
#include <istream>
namespace std {
typedef basic_istream<char, std::char_traits<char>> char_basic_istream;

View file

@ -0,0 +1,34 @@
commit 8076537a52bb026941f13f5542395aac69ef0825
Author: Austin Seipp <aseipp@pobox.com>
Date: Sat May 4 17:34:51 2019 -0500
cmake: add workarounds for NixOS-specific deficiencies [NixOS]
The NixOS debug builder hook adds '-Wa,--compress-debug-sections' to the
link flags (it actually adds it to the compiler flags, but the compiler
is used for linking, so...). This makes the compiler angry when -Werror
is passed, because it's unused at link-time (-Wa applies to the
assembler). Suppress this warning with -Wno-unused-command-line-argument
NB: we *could* use -Wno-error=unused-command-line-argument, but that
still results in warnings anyway, just not fatal ones. We'd like to
remove them all for the sake of the build output.
Signed-off-by: Austin Seipp <aseipp@pobox.com>
diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
index 03af9c10..7d059375 100644
--- a/cmake/ConfigureCompiler.cmake
+++ b/cmake/ConfigureCompiler.cmake
@@ -119,6 +119,11 @@ else()
else()
add_compile_options(-Werror)
endif()
+ if (CLANG)
+ # aseipp: NixOS hack
+ add_compile_options(-Wno-unused-command-line-argument)
+ add_link_options(-Wno-unused-command-line-argument)
+ endif()
add_compile_options($<$<BOOL:${GCC}>:-Wno-pragmas>)
add_compile_options(-Wno-error=format
-Wunused-variable

View file

@ -0,0 +1,80 @@
fast/AtomicBackupCorrectness.txt
fast/AtomicBackupToDBCorrectness.txt
fast/AtomicOps.txt
fast/AtomicOpsApiCorrectness.txt
fast/BackupCorrectness.txt
fast/BackupCorrectnessClean.txt
fast/BackupToDBCorrectness.txt
fast/BackupToDBCorrectnessClean.txt
fast/CloggedSideband.txt
fast/ConstrainedRandomSelector.txt
fast/CycleAndLock.txt
fast/CycleTest.txt
fast/FuzzApiCorrectness.txt
fast/FuzzApiCorrectnessClean.txt
fast/IncrementTest.txt
fast/InventoryTestAlmostReadOnly.txt
fast/InventoryTestSomeWrites.txt
fast/KillRegionCycle.txt
fast/LongStackWriteDuringRead.txt
fast/LowLatency.txt
fast/MemoryLifetime.txt
fast/MoveKeysCycle.txt
fast/RandomSelector.txt
fast/RandomUnitTests.txt
fast/SelectorCorrectness.txt
fast/Sideband.txt
fast/SidebandWithStatus.txt
fast/SwizzledRollbackSideband.txt
fast/SystemRebootTestCycle.txt
fast/TaskBucketCorrectness.txt
fast/TimeKeeperCorrectness.txt
fast/Unreadable.txt
fast/VersionStamp.txt
fast/Watches.txt
fast/WriteDuringRead.txt
fast/WriteDuringReadClean.txt
rare/CheckRelocation.txt
rare/ClogUnclog.txt
rare/CloggedCycleWithKills.txt
rare/ConflictRangeCheck.txt
rare/ConflictRangeRYOWCheck.txt
rare/CycleRollbackClogged.txt
rare/CycleWithKills.txt
rare/FuzzTest.txt
rare/InventoryTestHeavyWrites.txt
rare/LargeApiCorrectness.txt
rare/LargeApiCorrectnessStatus.txt
rare/RYWDisable.txt
rare/RandomReadWriteTest.txt
rare/SwizzledLargeApiCorrectness.txt
slow/ApiCorrectness.txt
slow/ApiCorrectnessAtomicRestore.txt
slow/ApiCorrectnessSwitchover.txt
slow/ClogWithRollbacks.txt
slow/CloggedCycleTest.txt
slow/CloggedStorefront.txt
slow/CommitBug.txt
slow/ConfigureTest.txt
slow/CycleRollbackPlain.txt
slow/DDBalanceAndRemove.txt
slow/DDBalanceAndRemoveStatus.txt
slow/FastTriggeredWatches.txt
slow/LowLatencyWithFailures.txt
slow/MoveKeysClean.txt
slow/MoveKeysSideband.txt
slow/RyowCorrectness.txt
slow/Serializability.txt
slow/SharedBackupCorrectness.txt
slow/SharedBackupToDBCorrectness.txt
slow/StorefrontTest.txt
slow/SwizzledApiCorrectness.txt
slow/SwizzledCycleTest.txt
slow/SwizzledDdBalance.txt
slow/SwizzledRollbackTimeLapse.txt
slow/SwizzledRollbackTimeLapseIncrement.txt
slow/VersionStampBackupToDB.txt
slow/VersionStampSwitchover.txt
slow/WriteDuringReadAtomicRestore.txt
slow/WriteDuringReadSwitchover.txt
slow/ddbalance.txt

View file

@ -5,6 +5,7 @@
, which, findutils, m4, gawk
, python, openjdk, mono, libressl
, ...
}:
let

View file

@ -3026,7 +3026,9 @@ in
inherit (fdbPackages)
foundationdb51
foundationdb52
foundationdb60;
foundationdb60
foundationdb61
;
foundationdb = foundationdb60;

View file

@ -5528,6 +5528,7 @@ in {
foundationdb51 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb51; };
foundationdb52 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb52; };
foundationdb60 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb60; };
foundationdb61 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb61; };
libtorrentRasterbar = (toPythonModule (pkgs.libtorrentRasterbar.override {
inherit python;