push.yml

name: master
on: push
jobs:
  test-internals:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    name: test-internals
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Internals tests
        run: python -m pytest modin/core/execution/dispatching/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py
      - run: python -m pytest modin/config/test
      - run: python -m pytest modin/test/test_envvar_catcher.py
      - run: python -m pytest modin/test/storage_formats/pandas/test_internals.py
      - run: python -m pytest modin/test/test_envvar_npartitions.py
      - run: python -m pytest modin/test/test_partition_api.py
      - run: python -m pytest modin/test/exchange/dataframe_protocol/base
      - run: python -m pytest modin/test/test_logging.py
      - uses: codecov/codecov-action@v2

  test-defaults:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        execution: [BaseOnPython]
    env:
      MODIN_MEMORY: 1000000000
      MODIN_TEST_DATASET_SIZE: "small"
    name: Test ${{ matrix.execution }} execution, Python 3.8
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: 3.8
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Install HDF5
        run: sudo apt update && sudo apt install -y libhdf5-dev
      - run: pytest -n 2 modin/experimental/xgboost/test/test_default.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_binary.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_default.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_indexing.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_iter.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_join_sort.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_map_metadata.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_reduce.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_udf.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_window.py --execution=${{ matrix.execution }}
      - run: pytest -n 2 modin/pandas/test/dataframe/test_pickle.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_series.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_rolling.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_concat.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_groupby.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_reshape.py --execution=${{ matrix.execution }}
      - run: python -m pytest -n 2 modin/pandas/test/test_general.py --execution=${{ matrix.execution }}
      - uses: codecov/codecov-action@v2

  test-omnisci:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    env:
      MODIN_EXPERIMENTAL: "True"
      MODIN_ENGINE: "native"
      MODIN_STORAGE_FORMAT: "omnisci"
    name: Test OmniSci storage format, Python 3.8
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - name: Setting up Modin environment
        uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin_on_omnisci
          environment-file: requirements/env_omnisci.yml
          python-version: 3.8
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Install HDF5
        run: sudo apt update && sudo apt install -y libhdf5-dev
      - run: pytest modin/test/storage_formats/omnisci/test_internals.py
      - run: pytest modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py
      - run: pytest modin/pandas/test/test_io.py::TestCsv
      - run: pytest modin/test/exchange/dataframe_protocol/test_general.py
      - run: pytest modin/test/exchange/dataframe_protocol/omnisci
      - uses: codecov/codecov-action@v2

  test-all:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8"]
        engine: ["python", "ray", "dask"]
    env:
      MODIN_ENGINE: ${{matrix.engine}}
      MODIN_MEMORY: 1000000000
      # Only test reading from SQL server and postgres on ubuntu for now.
      # Eventually, we should test on Windows, too, but we will have to set up
      # the servers differently.
      MODIN_TEST_READ_FROM_SQL_SERVER: true
      MODIN_TEST_READ_FROM_POSTGRES: true
    name: test-ubuntu (engine ${{matrix.engine}}, python ${{matrix.python-version}})
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - name: Install HDF5
        run: sudo apt update && sudo apt install -y libhdf5-dev
      - name: Set up postgres
        # Locally, specifying port 2345:5432 works, but 2345:2345 and 5432:5432 do not. This solution is from
        # https://stackoverflow.com/questions/36415654/cant-connect-docker-postgresql-9-3
        run: |
          sudo docker pull postgres
          sudo docker run --name some-postgres -e POSTGRES_USER=sa -e POSTGRES_PASSWORD=Strong.Pwd-123 -e POSTGRES_DB=postgres -d -p 2345:5432 postgres
      - run: pytest -n 2 modin/experimental/xgboost/test/test_default.py
      - run: pytest -n 2 modin/experimental/xgboost/test/test_xgboost.py
        if: matrix.engine == 'ray'
      - run: pytest -n 2 modin/experimental/xgboost/test/test_dmatrix.py
        if: matrix.engine == 'ray'
      - run: pytest -n 2 modin/pandas/test/dataframe/test_binary.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_default.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_indexing.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_iter.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_join_sort.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_map_metadata.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_reduce.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_udf.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_window.py
      - run: pytest -n 2 modin/pandas/test/dataframe/test_pickle.py
      - run: python -m pytest -n 2 modin/pandas/test/test_series.py
      - run: python -m pytest -n 2 modin/pandas/test/test_rolling.py
      - run: python -m pytest -n 2 modin/pandas/test/test_concat.py
        if: matrix.engine == 'python'
      - run: python -m pytest modin/pandas/test/test_concat.py # Ray and Dask versions fails with -n 2
        if: matrix.engine != 'python'
      - run: python -m pytest -n 2 modin/pandas/test/test_groupby.py
      - run: python -m pytest -n 2 modin/pandas/test/test_reshape.py
      - run: python -m pytest -n 2 modin/pandas/test/test_general.py
      - run: chmod +x ./.github/workflows/sql_server/set_up_sql_server.sh
      - run: ./.github/workflows/sql_server/set_up_sql_server.sh
      - run: python -m pytest modin/pandas/test/test_io.py
      - run: python -m pytest modin/experimental/pandas/test/test_io_exp.py
      - run: pytest modin/test/exchange/dataframe_protocol/test_general.py
      - run: pytest modin/test/exchange/dataframe_protocol/pandas/test_protocol.py
      - uses: codecov/codecov-action@v2

  test-windows:
    runs-on: windows-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8"]
        engine: ["ray", "dask"]
        test-task:
          - modin/pandas/test/dataframe/test_binary.py
          - modin/pandas/test/dataframe/test_default.py
          - modin/pandas/test/dataframe/test_indexing.py
          - modin/pandas/test/dataframe/test_iter.py
          - modin/pandas/test/dataframe/test_join_sort.py
          - modin/pandas/test/dataframe/test_map_metadata.py
          - modin/pandas/test/dataframe/test_reduce.py
          - modin/pandas/test/dataframe/test_udf.py
          - modin/pandas/test/dataframe/test_window.py
          - modin/pandas/test/dataframe/test_pickle.py
          - modin/pandas/test/test_series.py
          - modin/pandas/test/test_rolling.py
          - modin/pandas/test/test_concat.py
          - modin/pandas/test/test_groupby.py
          - modin/pandas/test/test_reshape.py
          - modin/pandas/test/test_general.py
          - modin/pandas/test/test_io.py
    env:
      MODIN_ENGINE: ${{matrix.engine}}
      MODIN_MEMORY: 1000000000
    name: test-windows
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 2
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
          auto-update-conda: true # this enable `use-only-tar-bz2` feature on Windows
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: python -m pytest ${{matrix.test-task}}
        if: matrix.test-task != 'modin/pandas/test/test_io.py'
      - timeout-minutes: 30
        run: python -m pytest modin/pandas/test/test_io.py
        if: matrix.test-task == 'modin/pandas/test/test_io.py'
      - uses: codecov/codecov-action@v2

  test-pyarrow:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8"]
    env:
      MODIN_STORAGE_FORMAT: pyarrow
      MODIN_EXPERIMENTAL: "True"
    name: test (pyarrow, python ${{matrix.python-version}})
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: sudo apt update && sudo apt install -y libhdf5-dev
      - run: python -m pytest modin/pandas/test/test_io.py::TestCsv

  test-spreadsheet:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.8"]
        engine: ["ray", "dask"]
    env:
      MODIN_EXPERIMENTAL: "True"
      MODIN_ENGINE: ${{matrix.engine}}
    name: test-spreadsheet (engine ${{matrix.engine}}, python ${{matrix.python-version}})
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1
      - uses: conda-incubator/setup-miniconda@v2
        with:
          activate-environment: modin
          environment-file: environment-dev.yml
          python-version: ${{matrix.python-version}}
          channel-priority: strict
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Conda environment
        run: |
          conda info
          conda list
      - run: python -m pytest modin/experimental/spreadsheet/test/test_general.py