diff --git a/buildconfigs/01-imagestream.yaml b/buildconfigs/01-imagestream.yaml index a5c76f1..47f8081 100644 --- a/buildconfigs/01-imagestream.yaml +++ b/buildconfigs/01-imagestream.yaml @@ -6,6 +6,10 @@ spec: lookupPolicy: local: true tags: + - name: okd-rpms + from: + kind: ImageStreamTag + name: release:artifacts - name: kuryr-cni from: kind: DockerImage diff --git a/buildconfigs/02-additional-imagestream.yaml b/buildconfigs/02-additional-imagestream.yaml index 1c919af..cf4ceda 100644 --- a/buildconfigs/02-additional-imagestream.yaml +++ b/buildconfigs/02-additional-imagestream.yaml @@ -1,3 +1,4 @@ +--- apiVersion: image.openshift.io/v1 kind: ImageStream metadata: @@ -14,11 +15,18 @@ spec: type: Local importPolicy: importMode: PreserveOriginal # Needs OKD 4.13 - - name: fedora36 - from: - kind: DockerImage - name: 'quay.io/fedora/fedora:36' - referencePolicy: - type: Local - importPolicy: - importMode: PreserveOriginal # Needs OKD 4.13 +# upstream already resides on fedora:37 (Dockerfile.okd -> Dockerfile.fcos) +# while scos is being built by Dockerfile.scos +# - name: fedora36 +# from: +# kind: DockerImage +# name: 'quay.io/fedora/fedora:36' +# referencePolicy: +# type: Local +# importPolicy: +# importMode: PreserveOriginal # Needs OKD 4.13 +--- +apiVersion: image.openshift.io/v1 +kind: ImageStream +metadata: + name: artifacts diff --git a/buildconfigs/03-forked-dockerfiles.yaml b/buildconfigs/03-forked-dockerfiles.yaml index f23cca1..a4e6ffe 100644 --- a/buildconfigs/03-forked-dockerfiles.yaml +++ b/buildconfigs/03-forked-dockerfiles.yaml @@ -3,6 +3,7 @@ kind: BuildConfig metadata: name: forked-dockerfiles labels: + builder-replacement: skip skip-release-branch: "true" skip-replace-cli: "true" spec: diff --git a/buildconfigs/05-base.yaml b/buildconfigs/05-base.yaml index b1e8412..b99cc98 100644 --- a/buildconfigs/05-base.yaml +++ b/buildconfigs/05-base.yaml @@ -3,6 +3,7 @@ kind: BuildConfig metadata: name: base labels: + builder-replacement: skip # TODO: do we need branch cut for this? skip-release-branch: "true" skip-replace-cli: "true" diff --git a/buildconfigs/06-cli.yaml b/buildconfigs/06-cli.yaml index 3f0ad60..4c37303 100644 --- a/buildconfigs/06-cli.yaml +++ b/buildconfigs/06-cli.yaml @@ -43,8 +43,7 @@ spec: - name: ARCHITECTURES value: "" - name: MAKE_ARGS - value: >- - -e --warn-undefined-variables + value: -e --warn-undefined-variables - name: INSTALL_PKGS value: "krb5-devel" - name: INSTALL_PKGS_RUNTIME @@ -57,6 +56,7 @@ spec: value: > OpenShift is a platform for developing, building, and deploying containerized applications. + # are these tag really necessary here? - name: TAGS value: openshift,cli - name: VERSIONS diff --git a/buildconfigs/10-baremetal-installer.yaml b/buildconfigs/10-baremetal-installer.yaml index d0443a2..810e9aa 100644 --- a/buildconfigs/10-baremetal-installer.yaml +++ b/buildconfigs/10-baremetal-installer.yaml @@ -25,6 +25,7 @@ spec: imageOptimizationPolicy: SkipLayers dockerfilePath: images/baremetal/Dockerfile.centos9 buildArgs: + # are these tag really necessary here? - name: "TAGS" value: "baremetal libvirt okd" from: diff --git a/buildconfigs/10-cli-artifacts.yaml b/buildconfigs/10-cli-artifacts.yaml index cde2a31..18d22cc 100644 --- a/buildconfigs/10-cli-artifacts.yaml +++ b/buildconfigs/10-cli-artifacts.yaml @@ -38,8 +38,7 @@ spec: - name: ARCHITECTURES value: "x86_64 aarch64 s390x ppc64le" - name: MAKE_ARGS - value: >- - cross-build --warn-undefined-variables + value: "cross-build --warn-undefined-variables" - name: INSTALL_PKGS value: "krb5-devel" - name: INSTALL_PKGS_RUNTIME @@ -52,6 +51,7 @@ spec: value: > OpenShift is a platform for developing, building, and deploying containerized applications. + # are these tag really necessary here? - name: TAGS value: openshift,cli - name: VERSIONS diff --git a/buildconfigs/10-hyperkube.yaml b/buildconfigs/10-hyperkube.yaml index 1cf46cb..8a7d314 100644 --- a/buildconfigs/10-hyperkube.yaml +++ b/buildconfigs/10-hyperkube.yaml @@ -57,6 +57,7 @@ spec: value: > OpenShift is a platform for developing, building, and deploying containerized applications. + # are these tag really necessary here? - name: TAGS value: openshift,hyperkube - name: VERSIONS diff --git a/buildconfigs/10-installer-artifact-amd64-lnx.yaml b/buildconfigs/10-installer-artifact-amd64-lnx.yaml new file mode 100644 index 0000000..d1bf7e4 --- /dev/null +++ b/buildconfigs/10-installer-artifact-amd64-lnx.yaml @@ -0,0 +1,24 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: installer-artifact-amd64-lnx + labels: + part-of-artifacts: installer +spec: + source: + type: Git + git: + uri: 'https://github.com/openshift/installer' + ref: release-4.12 + images: [] + strategy: + type: Docker + dockerStrategy: + buildArgs: + - name: "TARGET_ARCH" + value: "amd64" + - name: "TARGET_OS" + value: "linux" + output: + to: + name: 'artifacts:installer-amd64-lnx' diff --git a/buildconfigs/10-installer-artifact-amd64-mac.yaml b/buildconfigs/10-installer-artifact-amd64-mac.yaml new file mode 100644 index 0000000..c052ebd --- /dev/null +++ b/buildconfigs/10-installer-artifact-amd64-mac.yaml @@ -0,0 +1,24 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: installer-artifact-amd64-mac + labels: + part-of-artifacts: installer +spec: + source: + type: Git + git: + uri: 'https://github.com/openshift/installer' + ref: release-4.12 + images: [] + strategy: + type: Docker + dockerStrategy: + buildArgs: + - name: "TARGET_ARCH" + value: "amd64" + - name: "TARGET_OS" + value: "darwin" + output: + to: + name: 'artifacts:installer-amd64-mac' diff --git a/buildconfigs/10-installer-artifact-arm64-lnx.yaml b/buildconfigs/10-installer-artifact-arm64-lnx.yaml new file mode 100644 index 0000000..2015ba9 --- /dev/null +++ b/buildconfigs/10-installer-artifact-arm64-lnx.yaml @@ -0,0 +1,24 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: installer-artifact-arm64-lnx + labels: + part-of-artifacts: installer +spec: + source: + type: Git + git: + uri: 'https://github.com/openshift/installer' + ref: release-4.12 + images: [] + strategy: + type: Docker + dockerStrategy: + buildArgs: + - name: "TARGET_ARCH" + value: "arm64" + - name: "TARGET_OS" + value: "linux" + output: + to: + name: 'artifacts:installer-arm64-lnx' diff --git a/buildconfigs/10-installer-artifact-arm64-mac.yaml b/buildconfigs/10-installer-artifact-arm64-mac.yaml new file mode 100644 index 0000000..3001638 --- /dev/null +++ b/buildconfigs/10-installer-artifact-arm64-mac.yaml @@ -0,0 +1,24 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: installer-artifact-arm64-mac + labels: + part-of-artifacts: installer +spec: + source: + type: Git + git: + uri: 'https://github.com/openshift/installer' + ref: release-4.12 + images: [] + strategy: + type: Docker + dockerStrategy: + buildArgs: + - name: "TARGET_ARCH" + value: "arm64" + - name: "TARGET_OS" + value: "darwin" + output: + to: + name: 'artifacts:installer-arm64-mac' diff --git a/buildconfigs/10-installer-artifacts.yaml b/buildconfigs/10-installer-artifacts.yaml deleted file mode 100644 index 007ee06..0000000 --- a/buildconfigs/10-installer-artifacts.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: build.openshift.io/v1 -kind: BuildConfig -metadata: - name: installer-artifacts - labels: - resources-profile: high - change-build-arg-tags: "true" -spec: - source: - type: Git - git: - uri: 'https://github.com/openshift/installer' - ref: release-4.12 - contextDir: . - images: [] - strategy: - type: Docker - dockerStrategy: - imageOptimizationPolicy: SkipLayers - dockerfilePath: images/installer-artifacts/Dockerfile.rhel - buildArgs: - - name: "TAGS" - value: "okd" - from: - kind: "ImageStreamTag" - name: "release:base" - output: - to: - kind: ImageStreamTag - name: 'release:installer-artifacts' diff --git a/buildconfigs/10-installer.yaml b/buildconfigs/10-installer.yaml index d128f9b..d9d6db9 100644 --- a/buildconfigs/10-installer.yaml +++ b/buildconfigs/10-installer.yaml @@ -19,6 +19,7 @@ spec: imageOptimizationPolicy: SkipLayers dockerfilePath: images/installer/Dockerfile.ci buildArgs: + # are those tag really necessary here? seems "no" - name: "TAGS" value: "okd" from: diff --git a/buildconfigs/10-ironic.yaml b/buildconfigs/10-ironic.yaml index 0d70470..d8773bf 100644 --- a/buildconfigs/10-ironic.yaml +++ b/buildconfigs/10-ironic.yaml @@ -10,11 +10,11 @@ spec: ref: release-4.12 contextDir: . images: - - from: - kind: ImageStreamTag - name: 'tools:fedora36' - as: - - 'fedora:35' +# - from: +# kind: ImageStreamTag +# name: 'tools:fedora36' +# as: +# - 'fedora:35' - from: kind: ImageStreamTag name: 'tools:centos9' diff --git a/buildconfigs/10-machine-config-operator.yaml b/buildconfigs/10-machine-config-operator.yaml index d1c9280..62ad0b7 100644 --- a/buildconfigs/10-machine-config-operator.yaml +++ b/buildconfigs/10-machine-config-operator.yaml @@ -18,6 +18,7 @@ spec: imageOptimizationPolicy: SkipLayers dockerfilePath: Dockerfile buildArgs: + # are these tag really necessary here? seems "no" - name: "TAGS" value: "okd" from: diff --git a/buildconfigs/10-network-interface-bond-cni.yaml b/buildconfigs/10-network-interface-bond-cni.yaml index c74b678..319b235 100644 --- a/buildconfigs/10-network-interface-bond-cni.yaml +++ b/buildconfigs/10-network-interface-bond-cni.yaml @@ -1,7 +1,7 @@ apiVersion: build.openshift.io/v1 kind: BuildConfig metadata: - name: bond-cni + name: network-interface-bond-cni spec: source: type: Git diff --git a/buildconfigs/10-ovirt-installer.yaml b/buildconfigs/10-ovirt-installer.yaml index fd5d386..7a44527 100644 --- a/buildconfigs/10-ovirt-installer.yaml +++ b/buildconfigs/10-ovirt-installer.yaml @@ -19,6 +19,7 @@ spec: imageOptimizationPolicy: SkipLayers dockerfilePath: images/installer/Dockerfile.ci buildArgs: + # are those tag really necessary here? seems no - name: "TAGS" value: "okd" from: diff --git a/buildconfigs/20-artifacts.yaml b/buildconfigs/20-artifacts.yaml index ca5dd69..595bd14 100644 --- a/buildconfigs/20-artifacts.yaml +++ b/buildconfigs/20-artifacts.yaml @@ -3,8 +3,17 @@ kind: BuildConfig metadata: name: artifacts labels: + builder-replacement: skip + change-build-arg-tags: "true" + meta-for: installer + resources-profile: low skip-release-branch: "true" + skip-replace-cli: "true" spec: + output: + to: + kind: ImageStreamTag + name: 'release:artifacts' source: contextDir: . images: @@ -18,6 +27,31 @@ spec: name: 'release:hyperkube' as: - 'release:hyperkube' + - as: + - artifacts:installer-amd64-lnx + from: + kind: ImageStreamTag + name: artifacts:installer-amd64-lnx + - as: + - artifacts:installer-amd64-mac + from: + kind: ImageStreamTag + name: artifacts:installer-amd64-mac + - as: + - artifacts:installer-arm64-lnx + from: + kind: ImageStreamTag + name: artifacts:installer-arm64-lnx + - as: + - artifacts:installer-arm64-mac + from: + kind: ImageStreamTag + name: artifacts:installer-arm64-mac + #- from: + # kind: ImageStreamTag + # name: 'release:installer-artifacts' + # as: + # - 'release:installer-artifacts' - from: kind: ImageStreamTag name: 'release:forked-dockerfiles' @@ -27,12 +61,9 @@ spec: strategy: type: Docker dockerStrategy: - imageOptimizationPolicy: SkipLayers + buildArgs: [] dockerfilePath: Dockerfile.centos9 from: kind: "ImageStreamTag" name: "release:base" - output: - to: - kind: ImageStreamTag - name: 'release:artifacts' + imageOptimizationPolicy: SkipLayers diff --git a/buildconfigs/20-installer-artifacts.yaml b/buildconfigs/20-installer-artifacts.yaml new file mode 100644 index 0000000..7318789 --- /dev/null +++ b/buildconfigs/20-installer-artifacts.yaml @@ -0,0 +1,51 @@ +apiVersion: build.openshift.io/v1 +kind: BuildConfig +metadata: + name: installer-artifacts + labels: + builder-replacement: skip + change-build-arg-tags: "true" + meta-for: installer-artifacts + resources-profile: low + skip-release-branch: "true" +spec: + source: + type: Git + git: + uri: https://github.com/okd-project/okd-payload-pipeline + ref: main + contextDir: forks + images: + - as: + - artifacts:installer-amd64-lnx + from: + kind: ImageStreamTag + name: artifacts:installer-amd64-lnx + - as: + - artifacts:installer-amd64-mac + from: + kind: ImageStreamTag + name: artifacts:installer-amd64-mac + - as: + - artifacts:installer-arm64-lnx + from: + kind: ImageStreamTag + name: artifacts:installer-arm64-lnx + - as: + - artifacts:installer-arm64-mac + from: + kind: ImageStreamTag + name: artifacts:installer-arm64-mac + strategy: + type: Docker + dockerStrategy: + buildArgs: [] + dockerfilePath: artifacts/installer-artifacts-metaimage.dockerfile + from: + kind: "ImageStreamTag" + name: "release:base" + imageOptimizationPolicy: SkipLayers + output: + to: + kind: ImageStreamTag + name: 'release:installer-artifacts' diff --git a/buildconfigs/kustomization.yaml b/buildconfigs/kustomization.yaml index 1595d15..5585103 100644 --- a/buildconfigs/kustomization.yaml +++ b/buildconfigs/kustomization.yaml @@ -18,6 +18,15 @@ patches: target: kind: BuildConfig name: etcd + - patch: |- + - op: add + path: /spec/strategy/dockerStrategy/buildArgs + value: + - name: OCP_VERSION + value: "4.13" + target: + kind: BuildConfig + name: base # The next patch will replace the build-stage images used by the upstream repositories' Dockerfiles with the okd builder one. # In particular, each stage extending from an image contained in the `as` array below will start from the ImageStreamTag release:builder # In the future, we may change this to be more specific to the builder image features. E.g., we may need one builder image per golang version. @@ -49,6 +58,55 @@ patches: target: kind: BuildConfig labelSelector: "builder-replacement!=skip" +# +# part of installer artifacts build sequence +# + - patch: |- + - op: add + path: /metadata/labels + value: + builder-replacement: skip + change-build-arg-tags: "true" + part-of-artifacts: installer + resources-profile: high + skip-replace-cli: "true" + - op: add + path: /spec/output/to/kind + value: ImageStreamTag + - op: add + path: /spec/source/contextDir + value: "." + - op: add + path: /spec/source/images/- + value: + from: + kind: ImageStreamTag + name: 'release:forked-dockerfiles' + paths: + - sourcePath: /code/artifacts/installer-artifact.dockerfile + destinationDir: images/installer-artifacts + - op: add + path: /spec/strategy/dockerStrategy/imageOptimizationPolicy + value: SkipLayers + - op: add + path: /spec/strategy/dockerStrategy/dockerfilePath + value: images/installer-artifacts/installer-artifact.dockerfile + - op: add + path: /spec/source/dockerfile + value: | + FROM scratch + # - op: add + # path: /spec/strategy/dockerStrategy/from + # value: + # kind: "ImageStreamTag" + # name: "release:builder" + target: + kind: BuildConfig + labelSelector: "part-of-artifacts=installer" +# +# end of installer artifacts build sequence +# + - patch: |- - op: add path: "/spec/source/images/-" @@ -211,7 +269,11 @@ resources: - 10-ibm-vpc-block-csi-driver.yaml - 10-ibm-vpc-node-label-updater.yaml - 10-insights-operator.yaml - - 10-installer-artifacts.yaml +# - 10-installer-artifacts.yaml + - 10-installer-artifact-amd64-lnx.yaml + - 10-installer-artifact-amd64-mac.yaml + - 10-installer-artifact-arm64-lnx.yaml + - 10-installer-artifact-arm64-mac.yaml - 10-installer.yaml - 10-ironic-agent.yaml - 10-ironic-hardware-inventory-recorder.yaml @@ -284,6 +346,7 @@ resources: - 20-deployer.yaml - 20-haproxy-router.yaml - 20-machine-os-images.yaml + - 20-installer-artifacts.yaml - 20-ovn-kubernetes-base.yaml - 20-tools.yaml - 30-fedora-coreos.yaml diff --git a/forks/artifacts/Dockerfile.centos9 b/forks/artifacts/Dockerfile.centos9 index 9d19aa2..fccb0f9 100644 --- a/forks/artifacts/Dockerfile.centos9 +++ b/forks/artifacts/Dockerfile.centos9 @@ -1,7 +1,42 @@ +FROM fedora:37 AS crio + +ARG CRIO_VERSION=1.25 +ARG TAGS="" + +RUN \ + set -x ; \ + mkdir -p /rpms && \ + [ "$TAGS" = "fcos" ] || exit 0 ; \ + dnf install -y dnf-plugins-core && \ + dnf module enable -y cri-o:${CRIO_VERSION} && \ + dnf download -y cri-o cri-tools --destdir /rpms/$(uname -m) && \ + dnf clean all + FROM release:base +## is it necessary here? will release:installer-artifacts istag (offically at payload) be merged with the release:artifacts istag (currently not in payload) +#COPY --from=artifacts:installer-amd64-lnx /openshift-install /usr/share/openshift/linux_amd64/openshift-install +#COPY --from=artifacts:installer-arm64-lnx /openshift-install /usr/share/openshift/linux_arm64/openshift-install +#COPY --from=artifacts:installer-amd64-mac /openshift-install /usr/share/openshift/mac_arm64/openshift-install + COPY --from=release:cli-artifacts /srv/ /srv/ COPY --from=release:hyperkube /srv/ /srv/ +COPY --from=crio /rpms /tmp/rpms # TODO: the /srv/repo/ structure is slightly different than the origin-artifacts' structure and # this should lead to issues for the current consumers of this image: they should look for /srv/$(arch)/my_package.rpm + +RUN \ + set -x ; \ + if [ -d /tmp/rpms/$(uname -m) ] ; then \ + mkdir -p /rpms && \ + for r in /tmp/rpms/$(uname -m)/* ; do \ + mv -v $r /rpms/$(basename $r) ; \ + done ; \ + fi ; \ + if [ -d /srv/repo/$(uname -m) ] ; then \ + mkdir -p /rpms && \ + for r in /srv/repo/$(uname -m)/* ; do \ + ln -v $r /rpms/$(basename $r) ; \ + done ; \ + fi ; diff --git a/forks/artifacts/installer-artifact.dockerfile b/forks/artifacts/installer-artifact.dockerfile new file mode 100644 index 0000000..45f60eb --- /dev/null +++ b/forks/artifacts/installer-artifact.dockerfile @@ -0,0 +1,22 @@ +# +# it is part of the build sequence of installer-artifacts metapackage for a particular architecture and operating system +# +FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.19-openshift-4.12 AS builder + +ARG TAGS="" +ARG TARGET_OS=linux +ARG TARGET_ARCH=amd64 + +WORKDIR /go/src/github.com/openshift/installer + +COPY . . + +RUN \ + set -x ; \ + GOOS=${TARGET_OS} GOARCH=${TARGET_ARCH} DEFAULT_ARCH="$(go env GOHOSTARCH)" hack/build.sh +# +# copy result binary to blank image to avoid locat huge commit overhead and container registry fill +# +FROM scratch + +COPY --from=builder /go/src/github.com/openshift/installer/bin/openshift-install / \ No newline at end of file diff --git a/forks/artifacts/installer-artifacts-metaimage.dockerfile b/forks/artifacts/installer-artifacts-metaimage.dockerfile new file mode 100644 index 0000000..69bcffd --- /dev/null +++ b/forks/artifacts/installer-artifacts-metaimage.dockerfile @@ -0,0 +1,12 @@ +# +# this is an installer-artifacts final metaimage, which contains all openshift-installer binaries (except baremetal, is it needed to add here?) +# + +FROM registry.ci.openshift.org/ocp/4.12:installer + +COPY --from=artifacts:installer-amd64-lnx /openshift-install /usr/share/openshift/linux_amd64/openshift-install +COPY --from=artifacts:installer-arm64-lnx /openshift-install /usr/share/openshift/linux_arm64/openshift-install +COPY --from=artifacts:installer-amd64-mac /openshift-install /usr/share/openshift/mac_arm64/openshift-install + +# or mac_amd64? +COPY --from=artifacts:installer-arm64-mac /openshift-install /usr/share/openshift/mac/openshift-install diff --git a/pipelines/01-batch-build-task.yaml b/pipelines/01-batch-build-task.yaml index 46e8cc4..ce27835 100644 --- a/pipelines/01-batch-build-task.yaml +++ b/pipelines/01-batch-build-task.yaml @@ -1,3 +1,4 @@ +--- apiVersion: tekton.dev/v1beta1 kind: Task metadata: @@ -10,6 +11,9 @@ spec: - name: max_concurrent_builds type: string default: "8" + - name: build_outdated_only + type: string + default: "" steps: - name: builds image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest @@ -17,24 +21,93 @@ spec: env: - name: "MAX_PROCESSES" value: "$(params.max_concurrent_builds)" + - name: "BUILD_OUTDATED_ONLY" + value: "$(params.build_outdated_only)" timeout: 4h0m0s script: | #!/bin/bash - set -x - join() { + #set -x + + function is_build_outdated() { + local bc="$1" + + case $BUILD_OUTDATED_ONLY in + [Tt][Rr][Uu][Ee]|[Yy][Ee][Ss]) + ;; + *) + return 1 + ;; + esac + + local git_ref git_uri source_commit_id + local is_meta image_commit_id is_tag + + eval $(oc get bc $bc -o jsonpath='{"git_ref="}{.spec.source.git.ref}{"\ngit_uri="}{.spec.source.git.uri}{"\nis_meta="}{.metadata.labels.meta-for}{"\nis_tag="}{.spec.output.to.name}{"\n"}') + + [ -z "$is_meta" ] || return 1 + + # git ls-remote $git_uri -b $git_ref + source_commit_id=$(\ + curl \ + -d '0013command=ls-refs000800010008peel000bsymrefs0000' \ + --http1.1 \ + -H 'Accept: application/x-git-upload-pack-result' \ + -H 'Content-Type: application/x-git-upload-pack-request' \ + -H 'Git-Protocol: version=2' \ + -H 'Pragma: no-cache' \ + -H 'User-Agent: git/2.39.2' \ + -H 'Content-Length: 50' \ + "$git_uri/git-upload-pack" \ + 2> /dev/null | \ + awk "/refs\/[^\/]+\/${git_ref}$/ { print \$1; }" | \ + sed 's/....//') + + oc get istag $is_tag 1> /dev/null 2>&1 || return 1 + + istag_commit_id=$(oc describe istag $is_tag | awk -F= '/io.openshift.build.commit.id/ { print $2; }') + + if [ "$source_commit_id" = "$istag_commit_id" -a "${#source_commit_id}" = "40" ] ; then + echo "[$(get-cur-dat)] istag $is_tag is up-to-date (commit_id=$istag_commit_id)" + return 0 + fi + return 1 + } + + function get-cur-dat() { + echo -n $(date +%Y%m%d%H%M%S) + } + + function join() { wait -n code="$?" if [ $code -gt 0 ] && [ $code != "127" ]; then - # when $code = 127 the shell has no children processes. - exit "$code" + exit "$code" fi COUNTER=$(( COUNTER - 1 )) } + function start_build() { + local bc=$1 + local rc + local output + + echo "[$(get-cur-dat)] Build '$bc': start" + output=$(oc start-build $bc -w) + rc=$? + if [ "$rc" = "0" -o "$rc" = "127" ]; then + echo "[$(get-cur-dat)] Build '$bc': success($rc)" + else + echo -e $output + echo "[$(get-cur-dat)] Build '$bc': fail($rc)" + fi + exit $rc + } + COUNTER=0 for BUILD in "$@"; do - oc start-build "${BUILD}" -w & + is_build_outdated "${BUILD}" && continue + start_build "${BUILD}" & COUNTER=$(( COUNTER + 1 )) while [ $COUNTER -ge "$MAX_PROCESSES" ]; do join diff --git a/pipelines/build-from-scratch.yaml b/pipelines/build-from-scratch.yaml index 497e5a2..39f7f94 100644 --- a/pipelines/build-from-scratch.yaml +++ b/pipelines/build-from-scratch.yaml @@ -19,6 +19,14 @@ spec: - name: max_concurrent_builds type: string default: "8" + - name: build_outdated_only + default: "" + description: Build or not up-to-date images + type: string + - name: new-release-dry-run + description: Dry run of release task + type: string + default: "false" workspaces: - name: push-credentials tasks: @@ -26,6 +34,8 @@ spec: taskRef: name: batch-build params: + - name: build_outdated_only + value: "false" - name: max_concurrent_builds value: "1" - name: build_configs @@ -41,10 +51,14 @@ spec: taskRef: name: batch-build params: + - name: build_outdated_only + value: "$(params.build_outdated_only)" - name: max_concurrent_builds value: "$(params.max_concurrent_builds)" - name: build_configs value: + - agent-installer-api-server + - agent-installer-csr-approver - agent-installer-node-agent - agent-installer-orchestrator - alibaba-cloud-controller-manager @@ -66,10 +80,12 @@ spec: - azure-file-csi-driver-operator - azure-file-csi-driver - azure-machine-controllers + - baremetal-installer - baremetal-machine-controllers - baremetal-operator - baremetal-runtimecfg - branding + - cli-artifacts - cloud-credential-operator - cloud-network-config-controller - cluster-authentication-operator @@ -95,6 +111,7 @@ spec: - cluster-machine-approver - cluster-monitoring-operator - cluster-network-operator + - cluster-node-tuning-operator - cluster-openshift-apiserver-operator - cluster-openshift-controller-manager-operator - cluster-policy-controller @@ -103,10 +120,11 @@ spec: - cluster-update-keys - cluster-version-operator - configmap-reloader + - console - console-operator - container-networking-plugins - - contour-operator - contour + - contour-operator - coredns - csi-driver-manila-operator - csi-driver-manila @@ -133,6 +151,7 @@ spec: - gcp-pd-csi-driver-operator - gcp-pd-csi-driver - haproxy-router-base + - hyperkube - hypershift - ibm-cloud-controller-manager - ibm-vpc-block-csi-driver-operator @@ -140,33 +159,40 @@ spec: - ibm-vpc-node-label-updater - ibmcloud-machine-controllers - insights-operator + - installer + - ironic - ironic-agent + - ironic-hardware-inventory-recorder + - installer-artifact-amd64-lnx - ironic-machine-os-downloader - ironic-static-ip-manager - k8s-prometheus-adapter - keepalived-ipfailover + - installer-artifact-amd64-mac - kube-proxy - kube-rbac-proxy - kube-state-metrics - kube-storage-version-migrator - local-storage-static-provisioner + - installer-artifact-arm64-lnx + - libvirt-machine-controllers - machine-api-operator - machine-config-operator - machine-image-customization-controller - multus-admission-controller - multus-cni + - installer-artifact-arm64-mac - multus-networkpolicy - multus-route-override-cni - multus-whereabouts-ipam-cni - must-gather - - bond-cni + - network-interface-bond-cni - network-metrics-daemon - nutanix-machine-controllers - oauth-apiserver - oauth-proxy - oauth-server - oc-mirror - - cli-artifacts - openshift-apiserver - openshift-controller-manager - openshift-state-metrics @@ -177,10 +203,12 @@ spec: - openstack-machine-controllers - operator-lifecycle-manager - operator-marketplace + - operator-registry - ovirt-csi-driver-operator - ovirt-csi-driver - ovirt-installer - ovirt-machine-controllers + - ovn-kubernetes-base - pod - powervs-cloud-controller-manager - powervs-machine-controllers @@ -193,6 +221,7 @@ spec: - prometheus - route-controller-manager - service-ca-operator + - sdn - telemeter - thanos - vsphere-cloud-controller-manager @@ -201,35 +230,37 @@ spec: - vsphere-csi-driver-syncer - vsphere-csi-driver - vsphere-problem-detector - - cluster-node-tuning-operator - - ironic - - ironic-hardware-inventory-recorder - - libvirt-machine-controllers - - operator-registry - - sdn - - console - - baremetal-installer - - installer-artifacts - - installer - - hyperkube +# - installer-artifacts +# - name: meta-images +# timeout: "2h0m0s" +# runAfter: +# - batch-01 +# taskRef: +# name: batch-build +# params: +# - name: build_configs +# value: +# - installer-artifacts - name: batch-02 timeout: "2h0m0s" runAfter: - batch-01 +# - meta-images taskRef: name: batch-build params: + - name: build_outdated_only + value: "$(params.build_outdated_only)" - name: build_configs value: - - agent-installer-api-server - - agent-installer-csr-approver + - installer-artifacts - artifacts - - tools + - deployer - haproxy-router - - ovn-kubernetes - - ovn-kubernetes-base - machine-os-images - - deployer + - ovn-kubernetes + - ovn-kubernetes-microshift + - tools - name: batch-03 timeout: "1h0m0s" runAfter: @@ -237,11 +268,12 @@ spec: taskRef: name: batch-build params: + - name: build_outdated_only + value: "$(params.build_outdated_only)" - name: build_configs value: - - tests - network-tools - - ovn-kubernetes-microshift + - tests - name: new-release timeout: "2h0m0s" runAfter: @@ -259,3 +291,5 @@ spec: value: $(params.result-image) - name: release-image-name value: $(params.result-image-name) + - name: new-release-dry-run + value: $(params.new-release-dry-run) diff --git a/pipelines/full-rebuild.yaml b/pipelines/full-rebuild.yaml index 33ff823..d64ab30 100644 --- a/pipelines/full-rebuild.yaml +++ b/pipelines/full-rebuild.yaml @@ -255,7 +255,7 @@ spec: - multus-route-override-cni - multus-whereabouts-ipam-cni - must-gather - - bond-cni + - network-interface-bond-cni - network-metrics-daemon - nutanix-machine-controllers - oauth-apiserver diff --git a/variants/fcos/kustomization.yaml b/variants/fcos/kustomization.yaml index aaa24a5..267bcbc 100644 --- a/variants/fcos/kustomization.yaml +++ b/variants/fcos/kustomization.yaml @@ -9,21 +9,26 @@ resources: patches: - patch: |- - apiVersion: build.openshift.io/v1 - kind: BuildConfig - metadata: - name: not-used - labels: - change-build-arg-tags: "" - spec: - strategy: - dockerStrategy: - buildArgs: - - name: "TAGS" - value: "fcos" + - op: add + path: /spec/strategy/dockerStrategy/buildArgs/- + value: + name: "TAGS" + value: "fcos" target: labelSelector: "change-build-arg-tags==true" kind: BuildConfig + - patch: |- + - op: add + path: /spec/source/images/- + value: + from: + kind: ImageStreamTag + name: 'tools:fedora37' + as: + - 'fedora:37' + target: + kind: BuildConfig + name: artifacts - patch: |- - op: add path: "/spec/tags/-" @@ -34,6 +39,20 @@ patches: name: quay.io/fedora/fedora-coreos:next-devel referencePolicy: type: Source + # + # fc37 is necessary to build ironic and okd-rpms + - op: add + path: /spec/tags/- + value: + from: + kind: DockerImage + name: registry.fedoraproject.org/fedora:37 + #importPolicy: + # scheduled: true + # importMode: PreserveOriginal # Needs OKD 4.13 + name: fedora37 + referencePolicy: + type: Local target: kind: ImageStream name: tools @@ -50,7 +69,7 @@ patches: name: release - patch: |- - op: add - path: "/spec/tasks/3/params/0/value/-" + path: "/spec/tasks/3/params/1/value/-" value: fedora-coreos target: diff --git a/variants/scos/kustomization.yaml b/variants/scos/kustomization.yaml index f5e99d2..d8e3bbc 100644 --- a/variants/scos/kustomization.yaml +++ b/variants/scos/kustomization.yaml @@ -6,6 +6,13 @@ resources: - ../../pipelines patches: + - patch: |- + - op: add + path: /spec/strategy/dockerStrategy/dockerfilePath + value: Dockerfile.scos + target: + kind: BuildConfig + name: ironic - patch: |- apiVersion: build.openshift.io/v1 kind: BuildConfig