From b5cd9c0022798ec19bbb5e3006821dfa8a3fa16f Mon Sep 17 00:00:00 2001
From: Yee <2520865+yixinglu@users.noreply.github.com>
Date: Mon, 8 Nov 2021 22:08:47 +0800
Subject: [PATCH] Use GitHub Action to build nebula toolset automatically (#3)

* Use GitHub action to build nebula toolset automatically

* Cleanup ossutil URL

* Refactor

* login firstly

* Fix dockerfile

* Test toolset emulator building

* Fix this dir

* Remove llvm 8.0.0

* Only use centos-7 and debian-8

* Fix debian Dockerfile

* replace gcc 9.3.0 with 9.2.0

* Debug

* Fix data directory exists

* cancel in progress

* cancel in workflow group

* Fix buildx Dockerfile

* Restore

* cancel gcc 10.0.1

* Use self-hosted runners

* split different jobs

* Fix concurrency group

* fix concurrency
---
 .github/workflows/build.yml               | 88 +++++++++++++++++++++++
 .github/workflows/release.yml             |  4 +-
 docker/images/aarch64/centos-7.Dockerfile |  2 +-
 docker/images/aarch64/debian-8.Dockerfile |  2 +-
 docker/images/build-gcc.sh                |  7 +-
 docker/images/build-llvm.sh               |  7 +-
 docker/images/buildx/centos-7.Dockerfile  | 51 +++++++++++++
 docker/images/buildx/debian-8.Dockerfile  | 49 +++++++++++++
 docker/images/oss-upload.sh               | 12 +++-
 docker/images/run.sh                      | 13 ++++
 docker/images/x86_64/centos-7.Dockerfile  | 40 +++++------
 scripts/build-cmake                       |  2 +-
 scripts/build-llvm                        |  2 +-
 scripts/install-cmake                     |  2 +-
 scripts/install-gcc                       |  2 +-
 scripts/install-llvm                      |  2 +-
 scripts/nebula-gears-show-files           |  2 +-
 scripts/nebula-gears-uninstall            |  2 +-
 scripts/nebula-gears-update               |  2 +-
 19 files changed, 250 insertions(+), 41 deletions(-)
 create mode 100644 .github/workflows/build.yml
 create mode 100644 docker/images/buildx/centos-7.Dockerfile
 create mode 100644 docker/images/buildx/debian-8.Dockerfile
 create mode 100755 docker/images/run.sh

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 0000000..abea593
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,88 @@
+name: build
+
+on:
+  push:
+    branches:
+      - master
+
+defaults:
+  run:
+    shell: bash
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  gcc:
+    name: gcc
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - centos-7
+          - debian-8
+        arch:
+          - amd64
+          - arm64
+        version:
+          # - 7.5.0
+          # - 8.3.0
+          # - 9.1.0
+          - 9.2.0
+          # - 9.3.0
+          # - 10.1.0
+    steps:
+      - uses: actions/checkout@v1
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./docker/images/buildx/${{ matrix.os }}.Dockerfile
+          platforms: linux/${{ matrix.arch }}
+          tags: vesoft/toolset-build:${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.version }}
+          push: false
+          build-args: |
+            BUILD_GCC_VERSIONS=${{ matrix.version }}
+          secrets: |
+            "ossutilconfig=${{ secrets.OSS_UTIL_CONFIG }}"
+  llvm:
+    name: llvm
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - centos-7
+          - debian-8
+        arch:
+          - amd64
+          - arm64
+        version:
+          # - 9.0.0
+          - 10.0.0
+    steps:
+      - uses: actions/checkout@v1
+      - uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - uses: docker/setup-qemu-action@v1
+      - uses: docker/setup-buildx-action@v1
+      - uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: ./docker/images/buildx/${{ matrix.os }}.Dockerfile
+          platforms: linux/${{ matrix.arch }}
+          tags: vesoft/toolset-build:${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.version }}
+          push: false
+          build-args: |
+            BUILD_LLVM_VERSIONS=${{ matrix.version }}
+          secrets: |
+            "ossutilconfig=${{ secrets.OSS_UTIL_CONFIG }}"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c17b1dc..0f1454b 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,10 +1,10 @@
+name: release
+
 on:
   push:
     tags:
     - 'v*'
 
-name:
-
 jobs:
   build:
     name: Release
diff --git a/docker/images/aarch64/centos-7.Dockerfile b/docker/images/aarch64/centos-7.Dockerfile
index ca4b2c4..ccd554e 100644
--- a/docker/images/aarch64/centos-7.Dockerfile
+++ b/docker/images/aarch64/centos-7.Dockerfile
@@ -28,7 +28,7 @@ RUN yum install -y make \
 RUN yum install -y pxz || true
 
 ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
-ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64?spm=a2c63.p38356.879954.15.c0942454HuAZDI'
+ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64'
 ENV PACKAGE_DIR=/usr/src
 RUN curl -s ${NG_URL} | bash
 
diff --git a/docker/images/aarch64/debian-8.Dockerfile b/docker/images/aarch64/debian-8.Dockerfile
index 09818ab..e8a66ca 100644
--- a/docker/images/aarch64/debian-8.Dockerfile
+++ b/docker/images/aarch64/debian-8.Dockerfile
@@ -27,7 +27,7 @@ RUN apt-get install -y make \
 RUN apt-get install -y pxz --force-yes
 
 ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
-ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64?spm=a2c63.p38356.879954.15.c0942454HuAZDI'
+ENV OSS_UTIL_URL='https://gosspublic.alicdn.com/ossutil/1.7.0/ossutilarm64'
 ENV PACKAGE_DIR=/usr/src
 RUN set -o pipefail && curl -s ${NG_URL} | bash
 
diff --git a/docker/images/build-gcc.sh b/docker/images/build-gcc.sh
index 7e84b0a..88ef9f5 100644
--- a/docker/images/build-gcc.sh
+++ b/docker/images/build-gcc.sh
@@ -6,13 +6,12 @@ nebula-gears-update
 
 release="$(lsb_release -si) $(lsb_release -sr)"
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 versions=${BUILD_GCC_VERSIONS:-all}
 
 build-gcc --version=$versions
 
+[[ -d /data ]] && cp -v toolset-build/vesoft-gcc-*.sh /data/
 
-cp -v toolset-build/vesoft-gcc-*.sh /data
-
-[[ -n $OSS_ENDPOINT ]] && ${this_dir}/oss-upload.sh toolset toolset-build/vesoft-gcc-*.sh
+${this_dir}/oss-upload.sh toolset toolset-build/vesoft-gcc-*.sh
diff --git a/docker/images/build-llvm.sh b/docker/images/build-llvm.sh
index e398883..a1c2d43 100644
--- a/docker/images/build-llvm.sh
+++ b/docker/images/build-llvm.sh
@@ -8,7 +8,7 @@ nebula-gears-update
 
 release="$(lsb_release -si) $(lsb_release -sr)"
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 versions=${BUILD_LLVM_VERSIONS:-all}
 
@@ -19,7 +19,6 @@ source /opt/vesoft/toolset/cmake/enable
 
 build-llvm --version=$versions
 
+[[ -d /data ]] && cp -v toolset-build/vesoft-llvm-*.sh /data/
 
-cp -v toolset-build/vesoft-llvm-*.sh /data
-
-[[ -n $OSS_ENDPOINT ]] && ${this_dir}/oss-upload.sh toolset toolset-build/vesoft-llvm-*.sh
+${this_dir}/oss-upload.sh toolset toolset-build/vesoft-llvm-*.sh
diff --git a/docker/images/buildx/centos-7.Dockerfile b/docker/images/buildx/centos-7.Dockerfile
new file mode 100644
index 0000000..740b01d
--- /dev/null
+++ b/docker/images/buildx/centos-7.Dockerfile
@@ -0,0 +1,51 @@
+FROM centos:7
+
+ARG BUILD_LLVM_VERSIONS=""
+ARG BUILD_GCC_VERSIONS=""
+
+SHELL ["/bin/bash", "-c"]
+RUN yum update -y
+RUN yum install -y make \
+                   git \
+                   m4 \
+                   wget \
+                   unzip \
+                   bzip2 \
+                   xz \
+                   xz-devel \
+                   patch \
+                   python \
+                   python-devel \
+                   redhat-lsb-core \
+                   zlib-devel \
+                   gcc \
+                   gcc-c++ \
+                   libtool \
+                   autoconf \
+                   automake \
+                   bison \
+                   flex \
+                   gperf \
+                   gettext \
+                   epel-release
+
+RUN yum install -y pxz || true
+
+ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
+ENV OSS_UTIL_URL=http://gosspublic.alicdn.com/ossutil/1.7.0
+ENV PACKAGE_DIR=/usr/src
+RUN curl -s ${NG_URL} | bash
+
+RUN mkdir -p ${PACKAGE_DIR}
+WORKDIR ${PACKAGE_DIR}
+
+COPY . ${PACKAGE_DIR}
+
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-gcc.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-llvm.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/oss-upload.sh
+
+RUN [[ $(uname -m) = "aarch64" ]] && ARCH="arm"; wget -q -O /usr/bin/ossutil64 ${OSS_UTIL_URL}/ossutil${ARCH}64
+RUN chmod +x /usr/bin/ossutil64
+
+RUN --mount=type=secret,id=ossutilconfig,dst=/root/.ossutilconfig ${PACKAGE_DIR}/docker/images/run.sh
diff --git a/docker/images/buildx/debian-8.Dockerfile b/docker/images/buildx/debian-8.Dockerfile
new file mode 100644
index 0000000..797487d
--- /dev/null
+++ b/docker/images/buildx/debian-8.Dockerfile
@@ -0,0 +1,49 @@
+FROM debian:8
+
+ARG BUILD_LLVM_VERSIONS=""
+ARG BUILD_GCC_VERSIONS=""
+
+SHELL ["/bin/bash", "-c"]
+RUN apt-get update
+RUN apt-get install -y make \
+                       git \
+                       m4 \
+                       wget \
+                       unzip \
+                       curl \
+                       xz-utils \
+                       liblzma-dev \
+                       python-dev \
+                       patch \
+                       lsb-core \
+                       libz-dev \
+                       build-essential \
+                       libtool \
+                       automake \
+                       autoconf \
+                       autoconf-archive \
+                       autotools-dev \
+                       bison \
+                       flex \
+                       gperf \
+                       gettext --force-yes
+RUN apt-get install -y pxz
+
+ENV NG_URL=https://raw.githubusercontent.com/vesoft-inc/nebula-gears/master/install
+ENV OSS_UTIL_URL=http://gosspublic.alicdn.com/ossutil/1.7.0
+ENV PACKAGE_DIR=/usr/src
+RUN set -o pipefail && curl -s ${NG_URL} | bash
+
+RUN mkdir -p ${PACKAGE_DIR}
+WORKDIR ${PACKAGE_DIR}
+
+COPY . ${PACKAGE_DIR}
+
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-gcc.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/build-llvm.sh
+RUN chmod +x ${PACKAGE_DIR}/docker/images/oss-upload.sh
+
+RUN [[ $(uname -m) = "aarch64" ]] && ARCH="arm"; wget -q -O /usr/bin/ossutil64 ${OSS_UTIL_URL}/ossutil${ARCH}64
+RUN chmod +x /usr/bin/ossutil64
+
+RUN --mount=type=secret,id=ossutilconfig,dst=/root/.ossutilconfig ${PACKAGE_DIR}/docker/images/run.sh
diff --git a/docker/images/oss-upload.sh b/docker/images/oss-upload.sh
index ba00440..3c5169b 100755
--- a/docker/images/oss-upload.sh
+++ b/docker/images/oss-upload.sh
@@ -2,7 +2,17 @@
 
 [[ $# -lt 2 ]] && echo "$0 <subdir> <files...>" && exit 1
 
-CMD="ossutil64 -e ${OSS_ENDPOINT} -i ${OSS_ID} -k ${OSS_SECRET}"
+CMD=""
+if [[ ! -z "${OSS_ENDPOINT}" ]] && [[ ! -z "${OSS_ID}" ]] && [[ ! -z "${OSS_SECRET}" ]]; then
+    CMD="ossutil64 -e ${OSS_ENDPOINT} -i ${OSS_ID} -k ${OSS_SECRET}"
+elif [[ -f $HOME/.ossutilconfig ]]; then
+    CMD="ossutil64"
+elif [[ -f /run/secrets/ossutilconfig ]]; then
+    CMD="ossutil64 -c /run/secrets/ossutilconfig"
+else
+    echo "Could not find valid oss authentication configure file."
+    exit 1
+fi
 OSS_BASE=oss://nebula-graph
 OSS_SUBDIR=$1
 shift
diff --git a/docker/images/run.sh b/docker/images/run.sh
new file mode 100755
index 0000000..3c20703
--- /dev/null
+++ b/docker/images/run.sh
@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+set -e
+
+this_dir="$(cd "$(dirname "$0")" && pwd)"
+
+if [[ ! -z "${BUILD_GCC_VERSIONS}" ]]; then
+    $this_dir/build-gcc.sh
+fi
+
+if [[ ! -z "${BUILD_LLVM_VERSIONS}" ]]; then
+    $this_dir/build-llvm.sh
+fi
diff --git a/docker/images/x86_64/centos-7.Dockerfile b/docker/images/x86_64/centos-7.Dockerfile
index 493fce4..af7c8aa 100644
--- a/docker/images/x86_64/centos-7.Dockerfile
+++ b/docker/images/x86_64/centos-7.Dockerfile
@@ -3,26 +3,26 @@ SHELL ["/bin/bash", "-c"]
 RUN yum update -y
 RUN yum install -y make \
                    git \
-				   m4 \
-				   wget \
-				   unzip \
-				   bzip2 \
-				   xz \
-				   xz-devel \
-				   patch \
-				   python \
-				   python-devel \
-				   redhat-lsb-core \
-				   zlib-devel \
-				   gcc \
-				   gcc-c++ \
-				   libtool \
-				   autoconf \
-				   automake \
-				   bison \
-				   flex \
-				   gperf \
-				   gettext \
+                   m4 \
+                   wget \
+                   unzip \
+                   bzip2 \
+                   xz \
+                   xz-devel \
+                   patch \
+                   python \
+                   python-devel \
+                   redhat-lsb-core \
+                   zlib-devel \
+                   gcc \
+                   gcc-c++ \
+                   libtool \
+                   autoconf \
+                   automake \
+                   bison \
+                   flex \
+                   gperf \
+                   gettext \
                    epel-release
 
 RUN yum install -y pxz || true
diff --git a/scripts/build-cmake b/scripts/build-cmake
index 1d556cf..5d59d41 100755
--- a/scripts/build-cmake
+++ b/scripts/build-cmake
@@ -15,7 +15,7 @@ then
     exit $?
 fi
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 name=cmake
 version=3.15.7
diff --git a/scripts/build-llvm b/scripts/build-llvm
index a2c0b81..52980e5 100755
--- a/scripts/build-llvm
+++ b/scripts/build-llvm
@@ -15,7 +15,7 @@ then
     exit $?
 fi
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 [[ $# -ne 0 ]] && version=$(echo "$@" | sed 's;.*--version=(\S*).*;\1;p' -rn)
 
diff --git a/scripts/install-cmake b/scripts/install-cmake
index 37c4f6a..e8f3886 100755
--- a/scripts/install-cmake
+++ b/scripts/install-cmake
@@ -22,7 +22,7 @@ fi
 
 version=3.15.7
 url_base=https://oss-cdn.nebula-graph.com.cn/toolset
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 selected_libc=
 selected_archive=
 this_arch=$(uname -m)
diff --git a/scripts/install-gcc b/scripts/install-gcc
index 3df9f8d..edf4233 100755
--- a/scripts/install-gcc
+++ b/scripts/install-gcc
@@ -23,7 +23,7 @@ fi
 [[ $# -ne 0 ]] && version=$(echo "$@" | sed 's;.*--version=(\S*).*;\1;p' -rn)
 
 url_base=https://oss-cdn.nebula-graph.com.cn/toolset
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 version_files_dir=$this_dir/../share/nebula-gears
 [[ $# -ne 0 ]] && prefix=$(echo "$@" | sed 's;.*--prefix=(\S*).*;\1;p' -rn)
diff --git a/scripts/install-llvm b/scripts/install-llvm
index 4588a94..282d8b0 100755
--- a/scripts/install-llvm
+++ b/scripts/install-llvm
@@ -23,7 +23,7 @@ fi
 [[ $# -ne 0 ]] && version=$(echo "$@" | sed 's;.*--version=(\S*).*;\1;p' -rn)
 
 url_base=https://oss-cdn.nebula-graph.com.cn/toolset
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 this_arch=$(uname -m)
 
 version_files_dir=$this_dir/../share/nebula-gears
diff --git a/scripts/nebula-gears-show-files b/scripts/nebula-gears-show-files
index 01ee3d4..613d94a 100755
--- a/scripts/nebula-gears-show-files
+++ b/scripts/nebula-gears-show-files
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 bin_dir_name=$(basename $this_dir)
 [[ $bin_dir_name = bin ]] || exit 1
diff --git a/scripts/nebula-gears-uninstall b/scripts/nebula-gears-uninstall
index 706caca..55757f7 100755
--- a/scripts/nebula-gears-uninstall
+++ b/scripts/nebula-gears-uninstall
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 bin_dir_name=$(basename $this_dir)
 [[ $bin_dir_name = bin ]] || exit 1
diff --git a/scripts/nebula-gears-update b/scripts/nebula-gears-update
index 72cc1a3..19722b0 100755
--- a/scripts/nebula-gears-update
+++ b/scripts/nebula-gears-update
@@ -3,7 +3,7 @@
 set -e
 set -o pipefail
 
-this_dir=$(dirname $(readlink -f $0))
+this_dir="$(cd "$(dirname "$0")" && pwd)"
 
 bin_dir_name=$(basename $this_dir)
 [[ $bin_dir_name = bin ]] || exit 1