From 4594022ec312ccd3db4e29e1e7c658886de96d80 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Sat, 15 Feb 2020 11:06:41 +0100 Subject: [PATCH 1/4] pytest-lazy-fixture: init at 0.6.3 --- .../pytest-lazy-fixture/default.nix | 30 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 32 insertions(+) create mode 100644 pkgs/development/python-modules/pytest-lazy-fixture/default.nix diff --git a/pkgs/development/python-modules/pytest-lazy-fixture/default.nix b/pkgs/development/python-modules/pytest-lazy-fixture/default.nix new file mode 100644 index 00000000000..779b710f92b --- /dev/null +++ b/pkgs/development/python-modules/pytest-lazy-fixture/default.nix @@ -0,0 +1,30 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pytest +}: + +buildPythonPackage rec { + pname = "pytest-lazy-fixture"; + version = "0.6.3"; + + src = fetchPypi { + inherit pname version; + sha256 = "1b0hmnsxw4s2wf9pks8dg6dfy5cx3zcbzs8517lfccxsfizhqz8f"; + }; + + checkInputs = [ + pytest + ]; + + checkPhase = '' + pytest + ''; + + meta = with lib; { + description = "Helps to use fixtures in pytest.mark.parametrize"; + homepage = "https://github.com/pytest-dev/pytest-repeat"; + license = licenses.mit; + maintainers = with maintainers; [ tobim ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 148990650dd..3283197c2d2 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2332,6 +2332,8 @@ in { pytest-isort = callPackage ../development/python-modules/pytest-isort { }; + pytest-lazy-fixture = callPackage ../development/python-modules/pytest-lazy-fixture { }; + pytest-mpl = callPackage ../development/python-modules/pytest-mpl { }; pytest-mock = callPackage ../development/python-modules/pytest-mock { }; From 5ddadce27b98fc52c4ba82432513d8f38106b178 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Tue, 11 Feb 2020 11:57:42 +0100 Subject: [PATCH 2/4] arrow-cpp: 0.15.1 -> 0.16.0 This switches to explicitly enabling dependency lookups for optional features (they were not picked up before). It also enables building of the Parquet command line tools. --- .../libraries/arrow-cpp/default.nix | 41 ++++++++++++++----- .../arrow-cpp/jemalloc-disable-shared.patch | 16 ++++---- .../python-modules/pyarrow/default.nix | 4 +- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index 87684a4bbe4..e152848a70f 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -1,24 +1,24 @@ -{ stdenv, lib, fetchurl, fetchFromGitHub, fixDarwinDylibNames, autoconf, boost -, brotli, cmake, double-conversion, flatbuffers, gflags, glog, gtest, lz4, perl -, python, rapidjson, snappy, thrift, uriparser, which, zlib, zstd +{ stdenv, lib, fetchurl, fetchFromGitHub, fetchpatch, fixDarwinDylibNames, autoconf, boost +, brotli, cmake, flatbuffers, gflags, glog, gtest, lz4, perl +, python, rapidjson, snappy, thrift, which, zlib, zstd , enableShared ? true }: let parquet-testing = fetchFromGitHub { owner = "apache"; repo = "parquet-testing"; - rev = "a277dc4e55ded3e3ea27dab1e4faf98c112442df"; - sha256 = "1yh5a8l4ship36hwmgmp2kl72s5ac9r8ly1qcs650xv2g9q7yhnq"; + rev = "46c9e977f58f6c5ef1b81f782f3746b3656e5a8c"; + sha256 = "1z2s6zh58nf484s0yraw7b1aqgx66dn2wzp1bzv9ndq03msklwly"; }; in stdenv.mkDerivation rec { pname = "arrow-cpp"; - version = "0.15.1"; + version = "0.16.0"; src = fetchurl { url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz"; - sha256 = "1jbghpppabsix2rkxbnh41inj9lcxfz4q94p96xzxshh4g3mhb4s"; + sha256 = "1xdp1yni9i1cpml326s78qql1g832m800h7zjlqmk89983g94696"; }; sourceRoot = "apache-arrow-${version}/cpp"; @@ -35,6 +35,14 @@ in stdenv.mkDerivation rec { patches = [ # patch to fix python-test ./darwin.patch + # Adjust CMake target names to make -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON work. + # Remove this when updating to the next version. + (fetchpatch { + name = "arrow-use-upstream-cmake-target-names.patch"; + url = "https://github.com/apache/arrow/commit/396861b38d2f4e805db7c2ecd2c96fff0ca2678b.patch"; + sha256 = "0ki7nx858374anvwyi4szz5hgnnzv4fghdd05c38bzry9rfljgb1"; + stripLen = 1; + }) ] ++ lib.optionals (!enableShared) [ # The shared jemalloc lib is unused and breaks in static mode due to missing -fpic. ./jemalloc-disable-shared.patch @@ -48,7 +56,6 @@ in stdenv.mkDerivation rec { buildInputs = [ boost brotli - double-conversion flatbuffers gflags glog @@ -57,7 +64,6 @@ in stdenv.mkDerivation rec { rapidjson snappy thrift - uriparser zlib zstd python.pkgs.python @@ -71,15 +77,28 @@ in stdenv.mkDerivation rec { ''; cmakeFlags = [ + "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON" "-DARROW_BUILD_TESTS=ON" "-DARROW_DEPENDENCY_SOURCE=SYSTEM" - "-DARROW_PARQUET=ON" "-DARROW_PLASMA=ON" # Disable Python for static mode because openblas is currently broken there. "-DARROW_PYTHON=${if enableShared then "ON" else "OFF"}" - "-Duriparser_SOURCE=SYSTEM" + "-DARROW_USE_GLOG=ON" + "-DARROW_WITH_BROTLI=ON" + "-DARROW_WITH_LZ4=ON" + "-DARROW_WITH_SNAPPY=ON" + "-DARROW_WITH_ZLIB=ON" + "-DARROW_WITH_ZSTD=ON" + # Parquet options: + "-DARROW_PARQUET=ON" + "-DPARQUET_BUILD_EXECUTABLES=ON" + "-DTHRIFT_COMPILER=${thrift}/bin/thrift" + "-DTHRIFT_VERSION=${thrift.version}" ] ++ lib.optionals (!enableShared) [ "-DARROW_BUILD_SHARED=OFF" + "-DARROW_BOOST_USE_SHARED=OFF" + "-DARROW_GFLAGS_USE_SHARED=OFF" + "-DARROW_PROTOBUF_USE_SHARED=OFF" "-DARROW_TEST_LINKAGE=static" "-DOPENSSL_USE_STATIC_LIBS=ON" ] ++ lib.optional (!stdenv.isx86_64) "-DARROW_USE_SIMD=OFF"; diff --git a/pkgs/development/libraries/arrow-cpp/jemalloc-disable-shared.patch b/pkgs/development/libraries/arrow-cpp/jemalloc-disable-shared.patch index 0f7955724e1..564f6727388 100644 --- a/pkgs/development/libraries/arrow-cpp/jemalloc-disable-shared.patch +++ b/pkgs/development/libraries/arrow-cpp/jemalloc-disable-shared.patch @@ -1,11 +1,11 @@ diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake -@@ -1428,6 +1428,7 @@ if(ARROW_JEMALLOC) - "--with-jemalloc-prefix=je_arrow_" - "--with-private-namespace=je_arrow_private_" - "--without-export" -+ "--disable-shared" - # Don't override operator new() - "--disable-cxx" "--disable-libdl" - # See https://github.com/jemalloc/jemalloc/issues/1237 +@@ -1317,6 +1317,7 @@ if(ARROW_JEMALLOC) + "--with-jemalloc-prefix=je_arrow_" + "--with-private-namespace=je_arrow_private_" + "--without-export" ++ "--disable-shared" + # Don't override operator new() + "--disable-cxx" "--disable-libdl" + # See https://github.com/jemalloc/jemalloc/issues/1237 diff --git a/pkgs/development/python-modules/pyarrow/default.nix b/pkgs/development/python-modules/pyarrow/default.nix index 5590337951b..ab05f49ac38 100644 --- a/pkgs/development/python-modules/pyarrow/default.nix +++ b/pkgs/development/python-modules/pyarrow/default.nix @@ -1,4 +1,4 @@ -{ lib, buildPythonPackage, python, isPy3k, arrow-cpp, cmake, cython, futures, hypothesis, numpy, pandas, pytest, pkgconfig, setuptools_scm, six }: +{ lib, buildPythonPackage, python, isPy3k, arrow-cpp, cmake, cython, futures, hypothesis, numpy, pandas, pytest, pytest-lazy-fixture, pkgconfig, setuptools_scm, six }: let _arrow-cpp = arrow-cpp.override { inherit python; }; @@ -13,7 +13,7 @@ buildPythonPackage rec { nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ]; propagatedBuildInputs = [ numpy six ] ++ lib.optionals (!isPy3k) [ futures ]; - checkInputs = [ hypothesis pandas pytest ]; + checkInputs = [ hypothesis pandas pytest pytest-lazy-fixture ]; PYARROW_BUILD_TYPE = "release"; PYARROW_WITH_PARQUET = true; From 4b65f859c945b699e1ba91c62b6eb464ec314892 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Thu, 27 Feb 2020 12:49:44 -0500 Subject: [PATCH 3/4] pythonPackages.pytest-lazy-fixture: use pytestCheckHook --- .../python-modules/pytest-lazy-fixture/default.nix | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkgs/development/python-modules/pytest-lazy-fixture/default.nix b/pkgs/development/python-modules/pytest-lazy-fixture/default.nix index 779b710f92b..2b9c57b99d8 100644 --- a/pkgs/development/python-modules/pytest-lazy-fixture/default.nix +++ b/pkgs/development/python-modules/pytest-lazy-fixture/default.nix @@ -1,7 +1,7 @@ { lib , buildPythonPackage , fetchPypi -, pytest +, pytestCheckHook }: buildPythonPackage rec { @@ -14,13 +14,9 @@ buildPythonPackage rec { }; checkInputs = [ - pytest + pytestCheckHook ]; - checkPhase = '' - pytest - ''; - meta = with lib; { description = "Helps to use fixtures in pytest.mark.parametrize"; homepage = "https://github.com/pytest-dev/pytest-repeat"; From 56c2b7cb9a97f2ac86685c19488d32d91e5105cf Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Thu, 27 Feb 2020 12:50:23 -0500 Subject: [PATCH 4/4] arrow-cpp: fix RPATH on darwin Instead of patching test binaries with install_name_tool just set CMAKE_SKIP_BUILD_RPATH to OFF let cmake compile them with RPATH set correctly. Add CMAKE_INSTALL_RPATH to set correct RPATH for the installed executables for parquet tools. We could set it to an absolute path using "-DCMAKE_INSTALL_RPATH=${placeholder "out"}", but instead we use @loader_path to provide a relocable solution. --- pkgs/development/libraries/arrow-cpp/default.nix | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkgs/development/libraries/arrow-cpp/default.nix b/pkgs/development/libraries/arrow-cpp/default.nix index e152848a70f..50de0c88a5b 100644 --- a/pkgs/development/libraries/arrow-cpp/default.nix +++ b/pkgs/development/libraries/arrow-cpp/default.nix @@ -101,18 +101,17 @@ in stdenv.mkDerivation rec { "-DARROW_PROTOBUF_USE_SHARED=OFF" "-DARROW_TEST_LINKAGE=static" "-DOPENSSL_USE_STATIC_LIBS=ON" + ] ++ lib.optionals stdenv.isDarwin [ + "-DCMAKE_SKIP_BUILD_RPATH=OFF" # needed for tests + "-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables ] ++ lib.optional (!stdenv.isx86_64) "-DARROW_USE_SIMD=OFF"; doInstallCheck = true; PARQUET_TEST_DATA = if doInstallCheck then "${parquet-testing}/data" else null; installCheckInputs = [ perl which ]; - installCheckPhase = (lib.optionalString stdenv.isDarwin '' - for f in release/*test{,s}; do - install_name_tool -add_rpath "$out"/lib "$f" - done - '') - + (let + installCheckPhase = + let excludedTests = lib.optionals stdenv.isDarwin [ # Some plasma tests need to be patched to use a shorter AF_UNIX socket # path on Darwin. See https://github.com/NixOS/nix/pull/1085 @@ -122,7 +121,7 @@ in stdenv.mkDerivation rec { in '' ctest -L unittest -V \ --exclude-regex '^(${builtins.concatStringsSep "|" excludedTests})$' - ''); + ''; meta = { description = "A cross-language development platform for in-memory data";