perf: upgrade to latest conda, conda-build, and boa versions (#872)

bioconda · May 2, 2023 · 21a6452 · 21a6452
1 parent e7492c9
commit 21a6452
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 121 deletions.
diff --git a/.github/workflows/GithubActionTests.yml b/.github/workflows/GithubActionTests.yml
@@ -1,6 +1,6 @@
 name: PR Tests
 on: [pull_request]
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
@@ -16,77 +16,77 @@ jobs:
           - long_running_2
 
     steps:
-    - uses: actions/checkout@v1
+      - uses: actions/checkout@v1
 
-    - name: set path
-      run: echo "/opt/miniconda/bin" >> $GITHUB_PATH
+      - name: set path
+        run: echo "/opt/miniconda/bin" >> $GITHUB_PATH
 
-    - name: Install bioconda-utils
-      run: |
-        export BIOCONDA_DISABLE_BUILD_PREP=1
-        curl -L "https://raw.githubusercontent.com/bioconda/bioconda-common/master/install-and-set-up-conda.sh" > install-and-set-up-conda.sh
-        bash install-and-set-up-conda.sh
-        eval "$(conda shell.bash hook)"
-        mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
-        conda activate bioconda
-        python setup.py install
+      - name: Install bioconda-utils
+        run: |
+          export BIOCONDA_DISABLE_BUILD_PREP=1
+          curl -L "https://raw.githubusercontent.com/bioconda/bioconda-common/master/install-and-set-up-conda.sh" > install-and-set-up-conda.sh
+          bash install-and-set-up-conda.sh
+          eval "$(conda shell.bash hook)"
+          mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
+          conda activate bioconda
+          python setup.py install
 
-    - name: Build docker container
-      run: |
-        docker build -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest ./
-        docker history quay.io/bioconda/bioconda-utils-build-env-cos7:latest
-        docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info -a && conda list'
-        docker build -t quay.io/bioconda/bioconda-utils-test-env-cos7:latest -f ./Dockerfile.test ./
+      - name: Build docker container
+        run: |
+          docker build -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest ./
+          docker history quay.io/bioconda/bioconda-utils-build-env-cos7:latest
+          docker run --rm -t quay.io/bioconda/bioconda-utils-build-env-cos7:latest sh -lec 'type -t conda && conda info -a && conda list'
+          docker build -t quay.io/bioconda/bioconda-utils-test-env-cos7:latest -f ./Dockerfile.test ./
 
-    - name: Run tests '${{ matrix.py_test_marker }}'
-      run: |
-        eval "$(conda shell.bash hook)"
-        conda activate bioconda
-        if git diff --name-only origin/master...HEAD | grep -vE ^docs; then
-            py.test --durations=0 test/ -v --log-level=DEBUG --tb=native -m '${{ matrix.py_test_marker }}'
-        else
-            echo "Skipping pytest - only docs modified"
-        fi
+      - name: Run tests '${{ matrix.py_test_marker }}'
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate bioconda
+          if git diff --name-only origin/master...HEAD | grep -vE ^docs; then
+              py.test --durations=0 test/ -v --log-level=DEBUG --tb=native -m '${{ matrix.py_test_marker }}'
+          else
+              echo "Skipping pytest - only docs modified"
+          fi
   test-macosx:
     name: OSX tests
-    runs-on: macOS-latest
+    runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v1
+      - uses: actions/checkout@v1
 
-    - name: set path
-      run: echo "/opt/miniconda/bin" >> $GITHUB_PATH
+      - name: set path
+        run: echo "/opt/miniconda/bin" >> $GITHUB_PATH
 
-    - name: Install bioconda-utils
-      run: |
-        export BIOCONDA_DISABLE_BUILD_PREP=1
-        curl -L "https://raw.githubusercontent.com/bioconda/bioconda-common/master/install-and-set-up-conda.sh" > install-and-set-up-conda.sh
-        bash install-and-set-up-conda.sh
-        eval "$(conda shell.bash hook)"
-        mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
-        conda activate bioconda
-        python setup.py install
+      - name: Install bioconda-utils
+        run: |
+          export BIOCONDA_DISABLE_BUILD_PREP=1
+          curl -L "https://raw.githubusercontent.com/bioconda/bioconda-common/master/install-and-set-up-conda.sh" > install-and-set-up-conda.sh
+          bash install-and-set-up-conda.sh
+          eval "$(conda shell.bash hook)"
+          mamba create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
+          conda activate bioconda
+          python setup.py install
 
-    - name: Run tests
-      run: |
-        eval "$(conda shell.bash hook)"
-        conda activate bioconda
-        if git diff --name-only origin/master...HEAD | grep -vE ^docs; then
-            py.test --durations=0 test/ -v --log-level=DEBUG -k "not docker" --tb=native
-        else
-            echo "Skipping pytest - only docs modified"
-        fi
+      - name: Run tests
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate bioconda
+          if git diff --name-only origin/master...HEAD | grep -vE ^docs; then
+              py.test --durations=0 test/ -v --log-level=DEBUG -k "not docker" --tb=native
+          else
+              echo "Skipping pytest - only docs modified"
+          fi
 
   autobump-test:
     name: autobump test
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - name: Setup
-      uses: bioconda/bioconda-actions/bioconda_utils_setup_conda@master
-    - name: Install bioconda-utils
-      run: |
-        python setup.py install
-    - name: Test autobump
-      env:
-        AUTOBUMP_OPTS: --dry-run
-      uses: bioconda/bioconda-actions/bioconda_utils_autobump@master
+      - uses: actions/checkout@v1
+      - name: Setup
+        uses: bioconda/bioconda-actions/bioconda_utils_setup_conda@master
+      - name: Install bioconda-utils
+        run: |
+          python setup.py install
+      - name: Test autobump
+        env:
+          AUTOBUMP_OPTS: --dry-run
+        uses: bioconda/bioconda-actions/bioconda_utils_autobump@master
diff --git a/Dockerfile b/Dockerfile
@@ -21,9 +21,9 @@ RUN yum install -y mesa-libGL-devel \
 # This changes root's .condarc which ENTRYPOINT copies to /home/conda/.condarc later.
 RUN . /opt/conda/etc/profile.d/conda.sh && \
     conda config \
-      --prepend channels defaults \
-      --prepend channels bioconda \
-      --prepend channels conda-forge \
+    --prepend channels defaults \
+    --prepend channels bioconda \
+    --prepend channels conda-forge \
     && \
     { conda config --remove repodata_fns current_repodata.json 2> /dev/null || true ; } && \
     conda config --prepend repodata_fns repodata.json && \
@@ -38,8 +38,8 @@ RUN . /opt/conda/etc/profile.d/conda.sh  && conda activate base && \
     pip wheel . && \
     mkdir - /opt/bioconda-utils && \
     cp ./bioconda_utils-*.whl \
-        ./bioconda_utils/bioconda_utils-requirements.txt \
-        /opt/bioconda-utils/ \
+    ./bioconda_utils/bioconda_utils-requirements.txt \
+    /opt/bioconda-utils/ \
     && \
     chgrp -R lucky /opt/bioconda-utils && \
     chmod -R g=u /opt/bioconda-utils
@@ -49,15 +49,15 @@ COPY --from=build /opt/bioconda-utils /opt/bioconda-utils
 RUN . /opt/conda/etc/profile.d/conda.sh  && conda activate base && \
     # Make sure we get the (working) conda we want before installing the rest.
     sed -nE \
-        '/^conda([><!=~ ].+)?$/p' \
-        /opt/bioconda-utils/bioconda_utils-requirements.txt \
-        | xargs -r conda install --yes && \
-    conda install --yes --file /opt/bioconda-utils/bioconda_utils-requirements.txt && \
+    '/^conda([><!=~ ].+)?$/p' \
+    /opt/bioconda-utils/bioconda_utils-requirements.txt \
+    | xargs -r mamba install --yes && \
+    mamba install --yes --file /opt/bioconda-utils/bioconda_utils-requirements.txt && \
     pip install --no-deps --find-links /opt/bioconda-utils bioconda_utils && \
-    conda clean --yes --index --tarballs && \
+    mamba clean --yes --index --tarballs && \
     # Find files that are not already in group "lucky" and change their group and mode.
     find /opt/conda \
-      \! -group lucky \
-      -exec chgrp --no-dereference lucky {} + \
-      \! -type l \
-      -exec chmod g=u {} +
+    \! -group lucky \
+    -exec chgrp --no-dereference lucky {} + \
+    \! -type l \
+    -exec chmod g=u {} +
diff --git a/bioconda_utils/bioconda_utils-requirements.txt b/bioconda_utils/bioconda_utils-requirements.txt
@@ -4,9 +4,9 @@
 
 # basics
 python>=3.7
-conda=4.12.0
-boa=0.9.*
-conda-build=3.21.8
+conda=23.3.*
+boa=0.14.*
+conda-build=3.24.*
 conda-verify=3.1.*
 argh=0.26.*          # CLI
 colorlog=4.8.*       # Logging
@@ -17,7 +17,7 @@ networkx=2.*
 pandas=1.4.*
 numpy=1.19.*         # Avoid breaking pandas on OSX
 libblas=*=*openblas  # Avoid large mkl package (pulled in by pandas)
-boltons=18.*
+boltons=23.*
 jsonschema=3.2.*     # JSON schema verification
 simplejson           # Used by bioconda bot worker (NEEDED?)
 pyopenssl>=22.1      # Stay compatible with cryptography

diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py
@@ -132,10 +132,7 @@
 r"""
 FROM {docker_base_image}
 {proxies}
-RUN \
-    /opt/conda/bin/conda install -y conda={conda_ver} conda-build={conda_build_ver} \
-    && \
-    find /opt/conda \
+RUN find /opt/conda \
       \! -group lucky \
       -exec chgrp --no-dereference lucky {{}} + \
       \! -type l \
@@ -355,10 +352,8 @@ def _build_image(self):
         with open(os.path.join(build_dir, "Dockerfile"), 'w') as fout:
             fout.write(self.dockerfile_template.format(
                 docker_base_image=self.docker_base_image,
-                proxies=proxies,
-                conda_ver=conda.__version__,
-                conda_build_ver=conda_build.__version__)
-            )
+                proxies=proxies
+            ))
 
         logger.debug('Dockerfile:\n' + open(fout.name).read())
 

diff --git a/test/test_pkg_test.py b/test/test_pkg_test.py
@@ -94,38 +94,3 @@ def test_pkg_test_custom_base_image():
     built_packages = _build_pkg(RECIPE_CUSTOM_BASE)
     for pkg in built_packages:
         pkg_test.test_package(pkg, base_image='debian:latest')
-
-
-@pytest.mark.skipif(SKIP_OSX, reason="skipping on osx")
-def test_pkg_test_conda_image():
-    """
-    Check mulled-build test image has conda/mamba not older than outside build.
-    """
-    # Require versions at least at high as those used by bioconda-utils itself.
-    recipe = dedent(f"""
-        one:
-          meta.yaml: |
-            package:
-              name: test_pkg_test_conda_image
-              version: 0.1
-            requirements:
-              run:
-                - python
-                - setuptools
-            test:
-              commands:
-                - |
-                  python -c '
-                  import sys, os
-                  from pathlib import Path
-                  from pkg_resources import parse_version as v
-                  assert v("{conda_version}") <= v(Path(os.environ["CONDA_PREFIX"], "conda-version").read_text())
-                  assert v("{mamba_version}") <= v(Path(os.environ["CONDA_PREFIX"], "mamba-version").read_text())
-                  '
-          post-link.sh: |
-            conda --version | sed -n 's/^conda //p' > "${{PREFIX}}/conda-version"
-            mamba --version | sed -n 's/^mamba //p' > "${{PREFIX}}/mamba-version"
-    """)  # noqa: E501: line too long
-    built_packages = _build_pkg(recipe)
-    for pkg in built_packages:
-        pkg_test.test_package(pkg)