Merge pull request #38440 from veprbl/arrow_cpp_reopen

Add arrow-cpp, parquet-cpp, pythonPackages.pyarrow
This commit is contained in:
Matthew Justin Bauer 2018-04-17 10:01:46 -05:00 committed by GitHub
commit 09b5678dae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 159 additions and 3 deletions

View file

@ -0,0 +1,44 @@
{ stdenv, symlinkJoin, fetchurl, boost, brotli, cmake, flatbuffers, gtest, gflags, lz4, python, rapidjson, snappy, zlib, zstd }:
stdenv.mkDerivation rec {
name = "arrow-cpp-${version}";
version = "0.9.0";
src = fetchurl {
url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
sha256 = "16l91fixb5dgx3v6xc73ipn1w1hjgbmijyvs81j7ywzpna2cdcdy";
};
sourceRoot = "apache-arrow-${version}/cpp";
nativeBuildInputs = [ cmake ];
buildInputs = [ boost python.pkgs.python python.pkgs.numpy ];
preConfigure = ''
substituteInPlace cmake_modules/FindBrotli.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
substituteInPlace cmake_modules/FindLz4.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
'';
BROTLI_HOME = symlinkJoin { name="brotli-wrap"; paths = [ brotli.lib brotli.dev ]; };
FLATBUFFERS_HOME = flatbuffers;
GTEST_HOME = gtest;
GFLAGS_HOME = gflags;
LZ4_HOME = symlinkJoin { name="lz4-wrap"; paths = [ lz4 lz4.dev ]; };
RAPIDJSON_HOME = rapidjson;
SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; };
ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; };
ZSTD_HOME = zstd;
cmakeFlags = [
"-DARROW_PYTHON=ON"
];
meta = {
description = "A cross-language development platform for in-memory data";
homepage = https://arrow.apache.org/;
license = stdenv.lib.licenses.asl20;
platforms = stdenv.lib.platforms.unix;
maintainers = with stdenv.lib.maintainers; [ veprbl ];
};
}

View file

@ -0,0 +1,37 @@
{ stdenv, symlinkJoin, fetchurl, arrow-cpp, boost, cmake, gtest, snappy, thrift, zlib }:
stdenv.mkDerivation rec {
name = "parquet-cpp-${version}";
version = "1.4.0";
src = fetchurl {
url = "https://github.com/apache/parquet-cpp/archive/apache-${name}.tar.gz";
sha256 = "1kn7pjzi5san5f05qbl8l8znqsa3f9cq9bflfr4s2jfwr7k9p2aj";
};
nativeBuildInputs = [ cmake ];
buildInputs = [ boost ];
preConfigure = ''
substituteInPlace cmake_modules/FindThrift.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
substituteInPlace cmake_modules/FindSnappy.cmake --replace CMAKE_STATIC_LIBRARY CMAKE_SHARED_LIBRARY
'';
ARROW_HOME = arrow-cpp;
THRIFT_HOME = thrift;
GTEST_HOME = gtest;
SNAPPY_HOME = symlinkJoin { name="snappy-wrap"; paths = [ snappy snappy.dev ]; };
ZLIB_HOME = symlinkJoin { name="zlib-wrap"; paths = [ zlib.dev zlib.static ]; };
cmakeFlags = [
"-DPARQUET_BUILD_BENCHMARKS=OFF"
];
meta = {
description = "A C++ library to read and write the Apache Parquet columnar data format";
homepage = http://parquet.apache.org;
license = stdenv.lib.licenses.asl20;
platforms = stdenv.lib.platforms.unix;
maintainers = with stdenv.lib.maintainers; [ veprbl ];
};
}

View file

@ -13,14 +13,16 @@ stdenv.mkDerivation rec {
nativeBuildInputs = [ pkgconfig cmake ];
# detected by gcc7
NIX_CFLAGS_COMPILE = [ "-Wno-error=implicit-fallthrough" ];
preConfigure = ''
substituteInPlace CMakeLists.txt --replace "-Werror" ""
substituteInPlace example/CMakeLists.txt --replace "-Werror" ""
'';
meta = with lib; {
description = "Fast JSON parser/generator for C++ with both SAX/DOM style API";
homepage = "http://rapidjson.org/";
license = licenses.mit;
platforms = platforms.linux;
platforms = platforms.unix;
maintainers = with maintainers; [ cstrahan ];
};
}

View file

@ -0,0 +1,65 @@
{ lib, buildPythonPackage, python, isPy3k, fetchurl, arrow-cpp, cmake, cython, futures, numpy, pandas, pytest, pytestrunner, parquet-cpp, pkgconfig, setuptools_scm, six }:
let
_arrow-cpp = arrow-cpp.override { inherit python;};
_parquet-cpp = parquet-cpp.override { arrow-cpp = _arrow-cpp; };
in
buildPythonPackage rec {
pname = "pyarrow";
version = "0.9.0";
src = fetchurl {
url = "mirror://apache/arrow/arrow-${version}/apache-arrow-${version}.tar.gz";
sha256 = "16l91fixb5dgx3v6xc73ipn1w1hjgbmijyvs81j7ywzpna2cdcdy";
};
sourceRoot = "apache-arrow-${version}/python";
nativeBuildInputs = [ cmake cython pkgconfig setuptools_scm ];
propagatedBuildInputs = [ numpy six ] ++ lib.optionals (!isPy3k) [ futures ];
checkInputs = [ pandas pytest pytestrunner ];
PYARROW_BUILD_TYPE = "release";
PYARROW_CMAKE_OPTIONS = "-DCMAKE_INSTALL_RPATH=${ARROW_HOME}/lib;${PARQUET_HOME}/lib";
preBuild = ''
substituteInPlace CMakeLists.txt --replace "\''${ARROW_ABI_VERSION}" '"0.0.0"'
substituteInPlace CMakeLists.txt --replace "\''${ARROW_SO_VERSION}" '"0"'
# fix the hardcoded value
substituteInPlace cmake_modules/FindParquet.cmake --replace 'set(PARQUET_ABI_VERSION "1.0.0")' 'set(PARQUET_ABI_VERSION "${_parquet-cpp.version}")'
'';
preCheck = ''
rm pyarrow/tests/test_hdfs.py
# fails: "ArrowNotImplementedError: Unsupported numpy type 22"
substituteInPlace pyarrow/tests/test_feather.py --replace "test_timedelta_with_nulls" "_disabled"
# runs out of memory on @grahamcofborg linux box
substituteInPlace pyarrow/tests/test_feather.py --replace "test_large_dataframe" "_disabled"
# probably broken on python2
substituteInPlace pyarrow/tests/test_feather.py --replace "test_unicode_filename" "_disabled"
# fails "error: [Errno 2] No such file or directory: 'test'" because
# nix_run_setup invocation somehow manages to import deserialize_buffer.py
# when it is not intended to be imported at all
rm pyarrow/tests/deserialize_buffer.py
substituteInPlace pyarrow/tests/test_feather.py --replace "test_deserialize_buffer_in_different_process" "_disabled"
'';
ARROW_HOME = _arrow-cpp;
PARQUET_HOME = _parquet-cpp;
setupPyBuildFlags = ["--with-parquet" ];
meta = with lib; {
description = "A cross-language development platform for in-memory data";
homepage = https://arrow.apache.org/;
license = lib.licenses.asl20;
platforms = platforms.unix;
maintainers = with lib.maintainers; [ veprbl ];
};
}

View file

@ -8405,6 +8405,8 @@ with pkgs;
armadillo = callPackage ../development/libraries/armadillo {};
arrow-cpp = callPackage ../development/libraries/arrow-cpp {};
assimp = callPackage ../development/libraries/assimp { };
asio = callPackage ../development/libraries/asio { };
@ -10876,6 +10878,8 @@ with pkgs;
paperkey = callPackage ../tools/security/paperkey { };
parquet-cpp = callPackage ../development/libraries/parquet-cpp {};
pangoxsl = callPackage ../development/libraries/pangoxsl { };
pcaudiolib = callPackage ../development/libraries/pcaudiolib {

View file

@ -311,6 +311,10 @@ in {
pyamf = callPackage ../development/python-modules/pyamf { };
pyarrow = callPackage ../development/python-modules/pyarrow {
inherit (pkgs) arrow-cpp cmake pkgconfig;
};
pyatspi = disabledIf (!isPy3k) (callPackage ../development/python-modules/pyatspi { });
pyaxmlparser = callPackage ../development/python-modules/pyaxmlparser { };