Add Linux aarch64 support for bioconda-utils

Signed-off-by: Yikun Jiang <[email protected]>

.github/workflows/build-image.yml

@@ -14,7 +14,15 @@ jobs:
   build:
     name: Build image
     runs-on: ubuntu-20.04
-
+    strategy:
+      matrix:
+        include:
+          - dockerfile: Dockerfile.aarch64
+            arch: arm64
+            image: bioconda-utils-build-env-cos7-aarch64
+          - dockerfile: Dockerfile
+            arch: amd64
+            image: bioconda-utils-build-env-cos7
     steps:
     - uses: actions/checkout@v3
       with:
@@ -28,16 +36,22 @@ jobs:
         # printf %s "::set-output name=tag::${tag#v}"
         printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT
 
+    - name: Install qemu dependency
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y qemu-user-static
+
     - name: Build image
       id: buildah-build
       uses: redhat-actions/buildah-build@v2
       with:
-        image: bioconda-utils-build-env-cos7
+        image: ${{ matrix.image }}
+        arch: ${{ matrix.arch }}
         tags: >-
           latest
           ${{ steps.get-tag.outputs.tag }}
         dockerfiles: |
-          ./Dockerfile
+          ./${{ matrix.dockerfile }}
 
     - name: Test built image
       run: |

.github/workflows/release-please.yml

@@ -8,6 +8,15 @@ name: release-please
 jobs:
   release-please:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - dockerfile: Dockerfile.aarch64
+            arch: arm64
+            image: bioconda-utils-build-env-cos7-aarch64
+          - dockerfile: Dockerfile
+            arch: amd64
+            image: bioconda-utils-build-env-cos7
     steps:
 
       - uses: GoogleCloudPlatform/release-please-action@v2
@@ -28,17 +37,23 @@ jobs:
           tag=${{ steps.release.outputs.tag_name }}
           printf %s "::set-output name=tag::${tag#v}"
 
+      - name: Install qemu dependency
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y qemu-user-static
+
       - name: Build Image
         if: ${{ steps.release.outputs.release_created }}
         id: buildah-build
         uses: redhat-actions/buildah-build@v2
         with:
-          image: bioconda-utils-build-env-cos7
+          image: ${{ matrix.image }}
+          arch: ${{ matrix.arch }}
           tags: >-
             latest
             ${{ steps.get-tag.outputs.tag }}
           dockerfiles: |
-            ./Dockerfile
+            ./${{ matrix.dockerfile }}
 
       - name: Test Built Image
         if: ${{ steps.release.outputs.release_created }}

Dockerfile.aarch64

@@ -0,0 +1,78 @@
+# Specify the base image to support multi-arch images, such as
+# - 'quay.io/condaforge/linux-anvil-aarch64' for Linux aarch64
+# - 'quay.io/condaforge/linux-anvil-cos7-x86_64' for Linux x86_64
+ARG BASE_IMAGE=quay.io/condaforge/linux-anvil-aarch64
+
+FROM quay.io/condaforge/linux-anvil-aarch64 as base
+
+# Copy over C.UTF-8 locale from our base image to make it consistently available during build.
+COPY --from=quay.io/bioconda/base-glibc-busybox-bash /usr/lib/locale/C.UTF-8 /usr/lib/locale/C.UTF-8
+
+# Provide system deps unconditionally until we are able to offer per-recipe installs.
+# (Addresses, e.g., "ImportError: libGL.so.1" in tests directly invoked by conda-build.)
+# Also install packages that have been installed historically (openssh-client).
+RUN yum install -y mesa-libGL-devel \
+    && \
+    yum install -y openssh-clients \
+    && \
+    yum clean all && \
+    rm -rf /var/cache/yum/*
+
+# This changes root's .condarc which ENTRYPOINT copies to /home/conda/.condarc later.
+RUN . /opt/conda/etc/profile.d/conda.sh && \
+    conda config \
+      --prepend channels defaults \
+      --prepend channels bioconda \
+      --prepend channels conda-forge \
+    && \
+    { conda config --remove repodata_fns current_repodata.json 2> /dev/null || true ; } && \
+    conda config --prepend repodata_fns repodata.json && \
+    conda config --set channel_priority flexible && \
+    conda config --set auto_update_conda False
+
+FROM base as build
+WORKDIR /tmp/repo
+COPY . ./
+RUN . /opt/conda/etc/profile.d/conda.sh  && conda list
+RUN . /opt/conda/etc/profile.d/conda.sh  && conda activate base && \
+    pip wheel . && \
+    mkdir - /opt/bioconda-utils && \
+    cp ./bioconda_utils-*.whl \
+        ./bioconda_utils/bioconda_utils-requirements.txt \
+        /opt/bioconda-utils/ \
+    && \
+    chgrp -R lucky /opt/bioconda-utils && \
+    chmod -R g=u /opt/bioconda-utils
+
+# It's for multi-arch support. We do runtime involucro build here to solve
+# the chicken-and-egg conundrum. See also:
+# https://github.com/bioconda/bioconda-recipes/pull/40144
+FROM golang as involucro_build
+RUN wget https://github.com/involucro/involucro/archive/v1.1.2.tar.gz && \
+        tar -zxvf v1.1.2.tar.gz && \
+        mkdir -p src/github.com/involucro/ && \
+        mv involucro-1.1.2/ src/github.com/involucro/involucro && \
+        module_path="$( go env GOPATH )"/src/github.com/involucro/involucro/cmd/involucro && \
+        GO111MODULE=auto GOBIN="$( go env GOBIN )" go build -o ./bin/involucro -ldflags="-s -X github.com/involucro/involucro/app.version=1.1.2" "${module_path}"
+
+FROM base
+COPY --from=build /opt/bioconda-utils /opt/bioconda-utils
+COPY --from=involucro_build /go/bin/involucro /opt/conda/bin/
+RUN . /opt/conda/etc/profile.d/conda.sh  && conda activate base && \
+    # Make sure we get the (working) conda we want before installing the rest.
+    sed -nE \
+        '/^conda([><!=~ ].+)?$/p' \
+        /opt/bioconda-utils/bioconda_utils-requirements.txt \
+        | xargs -r conda install --yes && \
+    # Remove involucro line, build it in involucro_build stage
+    sed -i '/^involucro/d' /opt/bioconda-utils/bioconda_utils-requirements.txt && \
+    conda install --yes --file /opt/bioconda-utils/bioconda_utils-requirements.txt && \
+    pip install --no-deps --find-links /opt/bioconda-utils bioconda_utils && \
+    conda clean --yes --index --tarballs && \
+    # Find files that are not already in group "lucky" and change their group and mode.
+    find /opt/conda \
+      \! -group lucky \
+      -exec chgrp --no-dereference lucky {} + \
+      \! -type l \
+      -exec chmod g=u {} +
+

bioconda_utils/bioconda_utils-requirements.txt

@@ -9,16 +9,16 @@ boa=0.9.*
 conda-build=3.21.8
 conda-verify=3.1.*
 argh=0.26.*          # CLI
-colorlog=3.1.*       # Logging
+colorlog=4.8.*       # Logging
 tqdm>=4.26           # Progress monitor
 ruamel_yaml=0.15.*   # Recipe YAML parsing
 pyaml=17.12.*        # Faster YAML parser (deprecate?)
 networkx=2.*
-pandas=0.23.*
+pandas=1.4.*
 numpy=1.19.*         # Avoid breaking pandas on OSX
 libblas=*=*openblas  # Avoid large mkl package (pulled in by pandas)
 boltons=18.*
-jsonschema=2.6.*     # JSON schema verification
+jsonschema=3.2.*     # JSON schema verification
 simplejson           # Used by bioconda bot worker (NEEDED?)
 pyopenssl>=22.1      # Stay compatible with cryptography
 
@@ -28,11 +28,11 @@ conda-forge-pinning=2022.08.25.15.20.42
 # tools
 anaconda-client=1.6.*  # anaconda_upload
 involucro=1.1.*        # mulled test and container build
-skopeo=0.1.35          # docker upload
+skopeo=1.11.*          # docker upload
 git=2.*                # well - git
 
 # hosters - special regex not supported by RE
-regex=2018.08.29
+regex=2022.7.9
 
 # asyncio
 aiohttp=3.8.*      # HTTP lib
@@ -51,7 +51,7 @@ gidgethub=3.0.*           # githubhandler
 pyjwt>=2.4.0              # githubhandler (JWT signing), needs >=2.4.0, CVE-2022-29217
 
 # unknown
-beautifulsoup4=4.6.*
+beautifulsoup4=4.8.*
 galaxy-lib>=18.9.1
 jinja2>=2.10.1,<3
 markupsafe<2.1           # markupsafe 2.1 breaks jinja2
@@ -69,4 +69,4 @@ graphviz
 requests=2.22.*
 
 # merge handling
-pygithub
+pygithub

bioconda_utils/cli.py

@@ -423,12 +423,15 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False,
      than one worker, then make sure to give each a different offset!''')
 @arg('--keep-old-work', action='store_true', help='''Do not remove anything
 from environment, even after successful build and test.''')
+@arg('--docker-base-image', help='''Name of base image that can be used in\
+     **dockerfile_template**.''')
 @enable_logging()
 def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
           force=False, docker=None, mulled_test=False, build_script_template=None,
           pkg_dir=None, anaconda_upload=False, mulled_upload_target=None,
           build_image=False, keep_image=False, lint=False, lint_exclude=None,
-          check_channels=None, n_workers=1, worker_offset=0, keep_old_work=False):
+          check_channels=None, n_workers=1, worker_offset=0, keep_old_work=False,
+          docker_base_image='quay.io/bioconda/bioconda-utils-build-env-cos7:{}'.format(VERSION.replace('+', '_'))):
     cfg = utils.load_config(config)
     setup = cfg.get('setup', None)
     if setup:
@@ -454,6 +457,7 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
             use_host_conda_bld=use_host_conda_bld,
             keep_image=keep_image,
             build_image=build_image,
+            docker_base_image=docker_base_image
         )
     else:
         docker_builder = None

bioconda_utils/docker_utils.py

@@ -60,6 +60,8 @@
 import conda
 import conda_build
 
+from conda import exports as conda_exports
+
 from . import utils
 from . import __version__
 
@@ -90,9 +92,10 @@
 #
 # Note that if the directory didn't exist on the host, then the staging area
 # will exist in the container but will be empty.  Channels expect at least
-# a linux-64 and noarch directory within that directory, so we make sure it
-# exists before adding the channel.
+# a linux-64/linux-aarch64 and noarch directory within that directory, so we
+# make sure it exists before adding the channel.
 mkdir -p {self.container_staging}/linux-64
+mkdir -p {self.container_staging}/linux-aarch64
 mkdir -p {self.container_staging}/noarch
 touch {self.container_staging}/noarch/repodata.json
 conda config --add channels file://{self.container_staging} 2> >(
@@ -442,8 +445,9 @@ def build_recipe(self, recipe_dir, build_args, env, noarch=False):
 
         # Write build script to tempfile
         build_dir = os.path.realpath(tempfile.mkdtemp())
+        # conda_exports.subdir is {platform}-{arch} like: 'linux-64' 'linux-aarch64'
         script = self.build_script_template.format(
-            self=self, arch='noarch' if noarch else 'linux-64')
+            self=self, arch='noarch' if noarch else conda_exports.subdir)
         with open(os.path.join(build_dir, 'build_script.bash'), 'w') as fout:
             fout.write(script)
         build_script = fout.name