diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 6dd68c1273..b04e78b589 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -14,7 +14,15 @@ jobs: build: name: Build image runs-on: ubuntu-20.04 - + strategy: + matrix: + include: + - arch: arm64 + image: bioconda-utils-build-env-cos7-aarch64 + base_image: quay.io/condaforge/linux-anvil-aarch64 + - arch: amd64 + image: bioconda-utils-build-env-cos7 + base_image: quay.io/condaforge/linux-anvil-cos7-x86_64 steps: - uses: actions/checkout@v3 with: @@ -28,11 +36,19 @@ jobs: # printf %s "::set-output name=tag::${tag#v}" printf %s "tag=${tag#v}" >> $GITHUB_OUTPUT + - name: Install qemu dependency + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static + - name: Build image id: buildah-build uses: redhat-actions/buildah-build@v2 with: - image: bioconda-utils-build-env-cos7 + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} + build-args: | + BASE_IMAGE=${{ matrix.base_image }} tags: >- latest ${{ steps.get-tag.outputs.tag }} diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 86cb768a19..9c4c3d5e26 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -8,6 +8,15 @@ name: release-please jobs: release-please: runs-on: ubuntu-latest + strategy: + matrix: + include: + - arch: arm64 + image: bioconda-utils-build-env-cos7-aarch64 + base_image: quay.io/condaforge/linux-anvil-aarch64 + - arch: amd64 + image: bioconda-utils-build-env-cos7 + base_image: quay.io/condaforge/linux-anvil-cos7-x86_64 steps: - uses: GoogleCloudPlatform/release-please-action@v2 @@ -28,12 +37,20 @@ jobs: tag=${{ steps.release.outputs.tag_name }} printf %s "::set-output name=tag::${tag#v}" + - name: Install qemu dependency + run: | + sudo apt-get update + sudo apt-get install -y qemu-user-static + - name: Build Image if: ${{ steps.release.outputs.release_created }} id: buildah-build uses: redhat-actions/buildah-build@v2 with: - image: bioconda-utils-build-env-cos7 + image: ${{ matrix.image }} + arch: ${{ matrix.arch }} + build-args: | + BASE_IMAGE=${{ matrix.base_image }} tags: >- latest ${{ steps.get-tag.outputs.tag }} diff --git a/Dockerfile b/Dockerfile index 251af127dd..f509b73c3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,9 @@ -FROM quay.io/condaforge/linux-anvil-cos7-x86_64 as base +# Specify the base image to support multi-arch images, such as +# - 'quay.io/condaforge/linux-anvil-aarch64' for Linux aarch64 +# - 'quay.io/condaforge/linux-anvil-cos7-x86_64' for Linux x86_64 +ARG BASE_IMAGE=quay.io/condaforge/linux-anvil-cos7-x86_64 + +FROM ${BASE_IMAGE} as base # Copy over C.UTF-8 locale from our base image to make it consistently available during build. COPY --from=quay.io/bioconda/base-glibc-busybox-bash /usr/lib/locale/C.UTF-8 /usr/lib/locale/C.UTF-8 diff --git a/bioconda_utils/bioconda_utils-requirements.txt b/bioconda_utils/bioconda_utils-requirements.txt index a796722116..3cd499b43b 100644 --- a/bioconda_utils/bioconda_utils-requirements.txt +++ b/bioconda_utils/bioconda_utils-requirements.txt @@ -9,16 +9,16 @@ boa=0.9.* conda-build=3.21.8 conda-verify=3.1.* argh=0.26.* # CLI -colorlog=3.1.* # Logging +colorlog=4.8.* # Logging tqdm>=4.26 # Progress monitor ruamel_yaml=0.15.* # Recipe YAML parsing pyaml=17.12.* # Faster YAML parser (deprecate?) networkx=2.* -pandas=0.23.* +pandas=1.4.* numpy=1.19.* # Avoid breaking pandas on OSX libblas=*=*openblas # Avoid large mkl package (pulled in by pandas) boltons=18.* -jsonschema=2.6.* # JSON schema verification +jsonschema=3.2.* # JSON schema verification simplejson # Used by bioconda bot worker (NEEDED?) pyopenssl>=22.1 # Stay compatible with cryptography @@ -28,11 +28,11 @@ conda-forge-pinning=2022.08.25.15.20.42 # tools anaconda-client=1.6.* # anaconda_upload involucro=1.1.* # mulled test and container build -skopeo=0.1.35 # docker upload +skopeo=1.11.* # docker upload git=2.* # well - git # hosters - special regex not supported by RE -regex=2018.08.29 +regex=2022.7.9 # asyncio aiohttp=3.8.* # HTTP lib @@ -51,7 +51,7 @@ gidgethub=3.0.* # githubhandler pyjwt>=2.4.0 # githubhandler (JWT signing), needs >=2.4.0, CVE-2022-29217 # unknown -beautifulsoup4=4.6.* +beautifulsoup4=4.8.* galaxy-lib>=18.9.1 jinja2>=2.10.1,<3 markupsafe<2.1 # markupsafe 2.1 breaks jinja2 @@ -69,4 +69,4 @@ graphviz requests=2.22.* # merge handling -pygithub \ No newline at end of file +pygithub diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index b74701e4eb..7a66147723 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -426,13 +426,16 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False, than one worker, then make sure to give each a different offset!''') @arg('--keep-old-work', action='store_true', help='''Do not remove anything from environment, even after successful build and test.''') +@arg('--docker-base-image', help='''Name of base image that can be used in + Dockerfile template.''') @enable_logging() def build(recipe_folder, config, packages="*", git_range=None, testonly=False, force=False, docker=None, mulled_test=False, build_script_template=None, pkg_dir=None, anaconda_upload=False, mulled_upload_target=None, build_image=False, keep_image=False, lint=False, lint_exclude=None, check_channels=None, n_workers=1, worker_offset=0, keep_old_work=False, - mulled_conda_image=pkg_test.MULLED_CONDA_IMAGE): + mulled_conda_image=pkg_test.MULLED_CONDA_IMAGE, + docker_base_image='quay.io/bioconda/bioconda-utils-build-env-cos7:{}'.format(VERSION.replace('+', '_'))): cfg = utils.load_config(config) setup = cfg.get('setup', None) if setup: @@ -458,6 +461,7 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False, use_host_conda_bld=use_host_conda_bld, keep_image=keep_image, build_image=build_image, + docker_base_image=docker_base_image ) else: docker_builder = None diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index a43d864646..b6d21e8bed 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -60,6 +60,8 @@ import conda import conda_build +from conda import exports as conda_exports + from . import utils from . import __version__ @@ -90,9 +92,10 @@ # # Note that if the directory didn't exist on the host, then the staging area # will exist in the container but will be empty. Channels expect at least -# a linux-64 and noarch directory within that directory, so we make sure it -# exists before adding the channel. +# a linux-64/linux-aarch64 and noarch directory within that directory, so we +# make sure it exists before adding the channel. mkdir -p {self.container_staging}/linux-64 +mkdir -p {self.container_staging}/linux-aarch64 mkdir -p {self.container_staging}/noarch touch {self.container_staging}/noarch/repodata.json conda config --add channels file://{self.container_staging} 2> >( @@ -442,8 +445,9 @@ def build_recipe(self, recipe_dir, build_args, env, noarch=False): # Write build script to tempfile build_dir = os.path.realpath(tempfile.mkdtemp()) + # conda_exports.subdir is {platform}-{arch} like: 'linux-64' 'linux-aarch64' script = self.build_script_template.format( - self=self, arch='noarch' if noarch else 'linux-64') + self=self, arch='noarch' if noarch else conda_exports.subdir) with open(os.path.join(build_dir, 'build_script.bash'), 'w') as fout: fout.write(script) build_script = fout.name diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index 17ab445326..2db32ad463 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -49,6 +49,7 @@ from conda_build import api from conda.exports import VersionOrder +from conda.exports import subdir as conda_subdir from boa.cli.mambabuild import prepare as insert_mambabuild from jsonschema import validate @@ -1116,7 +1117,16 @@ def _filter_existing_packages(metas, check_channels): new_metas.append(meta) else: existing_metas.append(meta) - for divergent_build in (existing_pkg_builds - set(build_meta.keys())): + # Filter the existing_pkg_builds according to the native CPU architecture to avoid + # inaccurate divergent build results. + # + # For example, when a package has only `linux-64` arch type package, if we + # build on aarch64 machine, the `divergent_builds` will wrongly include the linux-64 + # one, we need to filter the non-native CPU architecture versions. + native_pkg_builds = { + x for x in existing_pkg_builds if x.subdir in (conda_subdir, 'noarch') + } + for divergent_build in (native_pkg_builds - set(build_meta.keys())): divergent_builds.add( '-'.join((pkg_key[0], pkg_key[1], divergent_build[1]))) return new_metas, existing_metas, divergent_builds