Skip to content

Latest commit

 

History

History
55 lines (50 loc) · 1.58 KB

README-pyarrow.md

File metadata and controls

55 lines (50 loc) · 1.58 KB

Developing arrow on Ubuntu 18.04 with CUDA

conda create -n pyarrow-dev
conda activate pyarrow-dev
conda install python numpy six setuptools cython pandas pytest \
      cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib \
      gflags brotli jemalloc lz4-c zstd \
      double-conversion glog autoconf hypothesis numba \
      clangdev=6 flake8 gtest gmock make \
      -c conda-forge
# clangdev=6 is required for clang-format
cd git/Quansight
git clone https://github.com/quansight/arrow.git

conda activate pyarrow-dev
cd arrow
export ARROW_BUILD_TYPE=release
export ARROW_BUILD_TOOLCHAIN=$CONDA_PREFIX
export ARROW_HOME=$CONDA_PREFIX
export PARQUET_HOME=$CONDA_PREFIX
export NUMBAPRO_LIBDEVICE=/usr/local/cuda-9.2/nvvm/libdevice
export NUMBAPRO_NVVM=/usr/local/cuda-9.2/nvvm/lib64/libnvvm.so
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
      -DARROW_PARQUET=OFF  -DARROW_PYTHON=ON  \
      -DARROW_PLASMA=OFF -DARROW_BUILD_TESTS=OFF \
      -DARROW_CUDA=ON \
      -DCLANG_FORMAT_BIN=`which clang-format` \
      ..
make -j3
make install
make format # after changing cpp/ files
cd ../../python
python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --with-cuda develop
py.test -sv pyarrow/

For development, use

export ARROW_BUILD_TYPE=debug
cmake -DCMAKE_BUILD_TYPE=$ARROW_BUILD_TYPE \
      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
      -DARROW_PARQUET=off  -DARROW_PYTHON=on  \
      -DARROW_PLASMA=off -DARROW_BUILD_TESTS=ON \
      -DARROW_CUDA=on \
      -DCLANG_FORMAT_BIN=`which clang-format` \
      ..