From 8735a8dfa3a9a3b60c63a903f0b2629dd1c5f9e1 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Tue, 2 Jul 2024 13:10:31 +0300 Subject: [PATCH 01/17] Add new nightly workflows --- .../workflows/monthly-copyright-update.yml | 1 + .../workflows/nightly-data-file-change.yml | 5 + .../nightly-documentation-update.yml | 18 +-- .github/workflows/nightly-package-update.yml | 1 + ...y-publish-main.yml => nightly-publish.yml} | 72 +++++------- ...r-to-main.yml => nightly-pull-request.yml} | 63 +++++------ ...-to-main.yml => nightly-pull-requests.yml} | 48 ++++---- .../workflows/nightly-submodule-update.yml | 19 ++-- generate-documentation.ps1 | 58 ++-------- monthly-copyright-update.ps1 | 33 ++---- nightly-data-file-change.ps1 | 67 +++-------- nightly-documentation-update.ps1 | 43 +++---- nightly-package-update.ps1 | 57 +++------- nightly-pr-to-main_build-and-test.ps1 | 105 ------------------ nightly-pr-to-main_complete.ps1 | 47 -------- nightly-pr-to-main_configure-pr.ps1 | 60 ---------- nightly-pr-to-main_get-prs.ps1 | 57 ---------- nightly-publish-main_build.ps1 | 48 -------- nightly-publish-main_configure.ps1 | 104 ----------------- nightly-publish-main_prebuild.ps1 | 64 ----------- nightly-publish-main_test.ps1 | 73 ------------ nightly-publish.build.ps1 | 32 ++++++ nightly-publish.configure.ps1 | 67 +++++++++++ ...package.ps1 => nightly-publish.package.ps1 | 49 +++----- nightly-publish.prebuild.ps1 | 40 +++++++ nightly-publish.test.ps1 | 44 ++++++++ nightly-pull-request.build-and-test.ps1 | 66 +++++++++++ ...ghtly-pull-request.compare-performance.ps1 | 37 ++---- nightly-pull-request.complete.ps1 | 18 +++ nightly-pull-request.configure-pr.ps1 | 35 ++++++ nightly-pull-requests.get-prs.ps1 | 29 +++++ nightly-submodule-update.ps1 | 54 ++------- python/run-integration-tests.ps1 | 2 +- python/run-unit-tests.ps1 | 2 +- steps/checkout-pr.ps1 | 52 +++------ steps/clone-repo.ps1 | 63 ++++------- steps/commit-changes.ps1 | 26 +---- steps/configure-git.ps1 | 14 ++- steps/fetch-hash-assets.ps1 | 2 +- steps/generate-documentation.ps1 | 56 +++++----- steps/get-next-package-version.ps1 | 34 ++---- steps/get-pull-requests.ps1 | 70 +++++------- steps/has-changed.ps1 | 35 +----- steps/merge-pr.ps1 | 42 ++----- steps/package-update-required.ps1 | 35 +----- steps/pull-request-to-main.ps1 | 45 -------- steps/pull-request.ps1 | 32 ++++++ steps/push-changes.ps1 | 43 ++----- steps/run-repo-script.ps1 | 58 ---------- steps/run-script.ps1 | 23 ++++ steps/update-sub-modules.ps1 | 28 +---- steps/update-tag.ps1 | 46 ++------ steps/upload-release-assets.ps1 | 50 +++------ 53 files changed, 773 insertions(+), 1499 deletions(-) rename .github/workflows/{nightly-publish-main.yml => nightly-publish.yml} (84%) rename .github/workflows/{nightly-pr-to-main.yml => nightly-pull-request.yml} (83%) rename .github/workflows/{nightly-prs-to-main.yml => nightly-pull-requests.yml} (73%) delete mode 100644 nightly-pr-to-main_build-and-test.ps1 delete mode 100644 nightly-pr-to-main_complete.ps1 delete mode 100644 nightly-pr-to-main_configure-pr.ps1 delete mode 100644 nightly-pr-to-main_get-prs.ps1 delete mode 100644 nightly-publish-main_build.ps1 delete mode 100644 nightly-publish-main_configure.ps1 delete mode 100644 nightly-publish-main_prebuild.ps1 delete mode 100644 nightly-publish-main_test.ps1 create mode 100644 nightly-publish.build.ps1 create mode 100644 nightly-publish.configure.ps1 rename nightly-publish-main_package.ps1 => nightly-publish.package.ps1 (57%) create mode 100644 nightly-publish.prebuild.ps1 create mode 100644 nightly-publish.test.ps1 create mode 100644 nightly-pull-request.build-and-test.ps1 rename nightly-pr-to-main_compare-performance.ps1 => nightly-pull-request.compare-performance.ps1 (72%) create mode 100644 nightly-pull-request.complete.ps1 create mode 100644 nightly-pull-request.configure-pr.ps1 create mode 100644 nightly-pull-requests.get-prs.ps1 delete mode 100644 steps/pull-request-to-main.ps1 create mode 100644 steps/pull-request.ps1 delete mode 100644 steps/run-repo-script.ps1 create mode 100644 steps/run-script.ps1 diff --git a/.github/workflows/monthly-copyright-update.yml b/.github/workflows/monthly-copyright-update.yml index cadab41e..f6152f07 100644 --- a/.github/workflows/monthly-copyright-update.yml +++ b/.github/workflows/monthly-copyright-update.yml @@ -49,6 +49,7 @@ jobs: -GitHubToken ${{ secrets.token }} ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` -GitHubUser ${{ inputs.user }} ` -GitHubEmail ${{ inputs.email }} ` -DryRun $DryRun diff --git a/.github/workflows/nightly-data-file-change.yml b/.github/workflows/nightly-data-file-change.yml index 08159a3e..d1861f37 100644 --- a/.github/workflows/nightly-data-file-change.yml +++ b/.github/workflows/nightly-data-file-change.yml @@ -22,6 +22,9 @@ on: required: false type: string default: false + common-ci-ref: + type: string + default: '' secrets: token: required: true @@ -43,6 +46,7 @@ jobs: with: repository: ${{ inputs.org-name }}/common-ci path: common + ref: ${{ inputs.common-ci-ref }} - name: Generate Accessors shell: pwsh @@ -53,6 +57,7 @@ jobs: -GitHubToken ${{ secrets.token }} ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` -DeviceDetectionKey ${{ secrets.device-detection-key }} ` -DeviceDetectionUrl "${{ secrets.device-detection-url }}" ` -GitHubUser ${{ inputs.user }} ` diff --git a/.github/workflows/nightly-documentation-update.yml b/.github/workflows/nightly-documentation-update.yml index 0bea73af..c42dd6f7 100644 --- a/.github/workflows/nightly-documentation-update.yml +++ b/.github/workflows/nightly-documentation-update.yml @@ -22,6 +22,9 @@ on: required: false type: string default: false + common-ci-ref: + type: string + default: '' separate-examples: required: false type: string @@ -31,26 +34,25 @@ on: required: true jobs: - nightly-documentation-update: + NightlyDocumentationUpdate: + name: Nightly Documentation Update runs-on: ubuntu-latest - + if: ${{ github.ref_name == 'main' }} # main only, for now steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Checkout reusable workflow dir + - name: Checkout Common uses: actions/checkout@v4 with: repository: ${{ inputs.org-name }}/common-ci path: common - + ref: ${{ inputs.common-ci-ref }} + - name: Generate Documentation shell: pwsh working-directory: ${{ github.workspace }}/common/ run: | $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) $SeparateExamples = [bool]::Parse( "${{ inputs.separate-examples}}" ) - . ${{ github.workspace }}/common/nightly-documentation-update.ps1 ` + ./nightly-documentation-update.ps1 ` -GitHubToken ${{ secrets.token }} ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` diff --git a/.github/workflows/nightly-package-update.yml b/.github/workflows/nightly-package-update.yml index 9af79f1a..0cb39c2b 100644 --- a/.github/workflows/nightly-package-update.yml +++ b/.github/workflows/nightly-package-update.yml @@ -54,6 +54,7 @@ jobs: -GitHubToken ${{ secrets.token }} ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` -RunId ${{ github.run_id }} ` -GitHubUser ${{ inputs.user }} ` -GitHubEmail ${{ inputs.email }} ` diff --git a/.github/workflows/nightly-publish-main.yml b/.github/workflows/nightly-publish.yml similarity index 84% rename from .github/workflows/nightly-publish-main.yml rename to .github/workflows/nightly-publish.yml index f7b27a7b..f7b302f3 100644 --- a/.github/workflows/nightly-publish-main.yml +++ b/.github/workflows/nightly-publish.yml @@ -1,35 +1,30 @@ -name: Nightly Publish Main +name: Nightly Publish on: workflow_call: inputs: repo-name: - required: true type: string - build-platform: required: true + build-platform: type: string + default: ubuntu-latest org-name: - required: false type: string default: 51degrees user: - required: false type: string default: Automation51D email: - required: false type: string default: "51DCI@51Degrees.com" dryrun: - required: false - type: string + type: boolean default: false cache-assets: - default: false type: boolean + default: false common-ci-ref: - required: false type: string default: '' secrets: @@ -74,16 +69,20 @@ on: DockerUser: PearlLicense: +defaults: + run: + shell: pwsh + working-directory: ./common + jobs: Configure: runs-on: ubuntu-latest outputs: - asset-cache-key: ${{ steps.hash-asset-cache-key.outputs.hash }} + asset_cache_key: ${{ steps.hash_asset_cache_key.outputs.hash }} options: ${{ steps.configure.outputs.options }} version: ${{ steps.configure.outputs.version }} update_required: ${{ steps.configure.outputs.update_required }} steps: - - name: Checkout Common uses: actions/checkout@v4 with: @@ -92,9 +91,8 @@ jobs: ref: ${{ inputs.common-ci-ref }} - name: Hash the Asset Cache Key - id: hash-asset-cache-key + id: hash_asset_cache_key if: inputs.cache-assets - shell: pwsh env: KEY: ${{ toJSON(secrets) }} run: | @@ -107,13 +105,11 @@ jobs: - name: Configure id: configure - shell: pwsh - working-directory: ${{ github.workspace }}/common/ run: | - . ${{ github.workspace }}/common/nightly-publish-main_configure.ps1 ` + ./nightly-publish.configure.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` - -Branch "${{ github.event_name == 'workflow_dispatch' && github.ref_name || '' }}" ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -GitHubOutput $Env:GITHUB_OUTPUT ` -GitHubUser ${{ inputs.user }} ` @@ -145,33 +141,31 @@ jobs: - name: PreBuild if: ${{ matrix.options.packagerequirement }} - shell: pwsh - working-directory: ${{ github.workspace }}/common/ env: SECRETS: ${{ toJSON(secrets) }} run: | $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) $Options = $(ConvertFrom-Json -AsHashtable '${{ toJSON(matrix.options) }}') $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) - . ${{ github.workspace }}/common/nightly-publish-main_prebuild.ps1 ` + ./nightly-publish.prebuild.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` - -Branch "${{ github.event_name == 'workflow_dispatch' && github.ref_name || '' }}" ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -Options $Options ` -GitHubUser ${{ inputs.user }} ` -GitHubEmail ${{ inputs.email }} ` -DryRun $DryRun - + - name: Upload Package Artifact uses: actions/upload-artifact@v4 if: ${{ success() && matrix.options.packagerequirement }} with: name: package_${{ matrix.options.name }} path: ${{ github.workspace }}/common/package-files - if-no-files-found: ignore + if-no-files-found: ignore include-hidden-files: true - + Build: runs-on: ${{ inputs.build-platform }} @@ -191,8 +185,6 @@ jobs: path: ${{ github.workspace }}/common/package-files - name: Build - shell: pwsh - working-directory: ${{ github.workspace }}/common/ env: SECRETS: ${{ toJSON(secrets) }} run: | @@ -200,10 +192,10 @@ jobs: $Options = @{} $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) $Options.Add("Version", "${{ needs.configure.outputs.version }}") - . ${{ github.workspace }}/common/nightly-publish-main_build.ps1 ` + ./nightly-publish.build.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` - -Branch "${{ github.event_name == 'workflow_dispatch' && github.ref_name || '' }}" ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -Options $Options ` -GitHubUser ${{ inputs.user }} ` @@ -227,7 +219,7 @@ jobs: options: ${{ fromJSON(needs.configure.outputs.options ) }} name: Test - ${{ matrix.options.name }} runs-on: ${{ matrix.options.image }} - + steps: - name: Checkout Common if: ${{ matrix.options.packagerequirement }} @@ -255,23 +247,21 @@ jobs: if: inputs.cache-assets uses: actions/cache/restore@v4 with: - key: ${{ needs.configure.outputs.asset-cache-key }} + key: ${{ needs.configure.outputs.asset_cache_key }} path: ${{ github.workspace }}/common/assets - name: Test - shell: pwsh if: ${{ success() && matrix.options.packagerequirement }} - working-directory: ${{ github.workspace }}/common/ env: SECRETS: ${{ toJSON(secrets) }} run: | $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) $Options = $(ConvertFrom-Json -AsHashtable '${{ toJSON(matrix.options) }}') $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) - . ${{ github.workspace }}/common/nightly-publish-main_test.ps1 ` + ./nightly-publish.test.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` - -Branch "${{ github.event_name == 'workflow_dispatch' && github.ref_name || '' }}" ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -Options $Options ` -GitHubUser ${{ inputs.user }} ` @@ -282,7 +272,7 @@ jobs: if: inputs.cache-assets && always() uses: actions/cache/save@v4 with: - key: ${{ needs.configure.outputs.asset-cache-key }} + key: ${{ needs.configure.outputs.asset_cache_key }} path: ${{ github.workspace }}/common/assets - name: Publish Test Results @@ -293,8 +283,8 @@ jobs: name: ${{ matrix.options.name }} skip-unit: true skip-performance: true - - Package: + + Package: runs-on: ${{ inputs.build-platform }} needs: [ Test, Configure ] if: ${{ success() && needs.configure.outputs.update_required == 'true' }} @@ -315,9 +305,7 @@ jobs: path: ${{ github.workspace }}/common/package - name: Package - shell: pwsh if: ${{ success() }} - working-directory: ${{ github.workspace }}/common/ env: SECRETS: ${{ toJSON(secrets) }} run: | @@ -325,10 +313,10 @@ jobs: $Options = @{} $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) $Options.Add("Version", "${{ needs.configure.outputs.version }}") - . ${{ github.workspace }}/common/nightly-publish-main_package.ps1 ` + ./nightly-publish.package.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` - -Branch "${{ github.event_name == 'workflow_dispatch' && github.ref_name || '' }}" ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -Options $Options ` -GitHubUser ${{ inputs.user }} ` diff --git a/.github/workflows/nightly-pr-to-main.yml b/.github/workflows/nightly-pull-request.yml similarity index 83% rename from .github/workflows/nightly-pr-to-main.yml rename to .github/workflows/nightly-pull-request.yml index 8577d1f1..bbab722a 100644 --- a/.github/workflows/nightly-pr-to-main.yml +++ b/.github/workflows/nightly-pull-request.yml @@ -1,4 +1,4 @@ -name: Nightly PR to Main +name: Nightly Pull Request on: workflow_call: @@ -10,26 +10,21 @@ on: required: true type: number org-name: - required: false type: string default: 51degrees user: - required: false type: string default: Automation51D email: - required: false type: string default: "51DCI@51Degrees.com" dryrun: - required: false type: string default: false cache-assets: default: false type: boolean common-ci-ref: - required: false type: string default: '' secrets: @@ -62,15 +57,19 @@ on: UsePublishTests: Pearl.License: +defaults: + run: + shell: pwsh + working-directory: ./common + jobs: Configure: - name : Configure - ${{ inputs.pull-request-id }} + name : Configure - ${{ inputs.pull-request-id }} runs-on: ubuntu-latest outputs: options: ${{ steps.configure.outputs.options }} - asset-cache-key: ${{ steps.hash-asset-cache-key.outputs.hash }} + asset_cache_key: ${{ steps.hash_asset_cache_key.outputs.hash }} steps: - - name: Checkout Common uses: actions/checkout@v4 with: @@ -79,9 +78,8 @@ jobs: ref: ${{ inputs.common-ci-ref }} - name: Hash the Asset Cache Key - id: hash-asset-cache-key + id: hash_asset_cache_key if: inputs.cache-assets - shell: pwsh env: KEY: ${{ toJSON(secrets) }} run: | @@ -93,27 +91,25 @@ jobs: "hash=$(Get-Date -Format FileDate)-$((Get-FileHash -InputStream $stringAsStream).Hash)" | Out-File $env:GITHUB_OUTPUT -Append - name: Configure - shell: pwsh id: configure - working-directory: ${{ github.workspace }}/common/ run: | - . ${{ github.workspace }}/common/nightly-pr-to-main_configure-pr.ps1 ` + ./nightly-pull-request.configure-pr.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -GitHubOutput $Env:GITHUB_OUTPUT ` -PullRequestId ${{ inputs.pull-request-id }} ` -GitHubUser ${{ inputs.user }} ` -GitHubEmail ${{ inputs.email }} - Build-and-Test: + BuildAndTest: needs: Configure strategy: matrix: options: ${{ fromJSON(needs.configure.outputs.options ) }} name: Build and Test - ${{ inputs.pull-request-id }} - ${{ matrix.options.name }} runs-on: ${{ matrix.options.image }} - steps: - name: Checkout Common uses: actions/checkout@v4 @@ -133,22 +129,21 @@ jobs: if: inputs.cache-assets uses: actions/cache/restore@v4 with: - key: ${{ needs.configure.outputs.asset-cache-key }} + key: ${{ needs.configure.outputs.asset_cache_key }} path: ${{ github.workspace }}/common/assets - name: Build and Test id: build-and-test - shell: pwsh - working-directory: ${{ github.workspace }}/common/ env: SECRETS: ${{ toJSON(secrets) }} run: | $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) $Options = $(ConvertFrom-Json -AsHashtable '${{ toJSON(matrix.options) }}') $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) - . ${{ github.workspace }}/common/nightly-pr-to-main_build-and-test.ps1 ` + ./nightly-pull-request.build-and-test.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -GitHubOutput $Env:GITHUB_OUTPUT ` -PullRequestId ${{ inputs.pull-request-id }} ` @@ -161,7 +156,7 @@ jobs: if: inputs.cache-assets && always() uses: actions/cache/save@v4 with: - key: ${{ needs.configure.outputs.asset-cache-key }} + key: ${{ needs.configure.outputs.asset_cache_key }} path: ${{ github.workspace }}/common/assets - name: Publish Test Results @@ -181,7 +176,7 @@ jobs: path: ${{ github.workspace }}/common/${{ inputs.repo-name }}/test-results/performance-summary/results_*.json if-no-files-found: ignore include-hidden-files: true - + - name: Upload Artifacts uses: actions/upload-artifact@v4 with: @@ -191,11 +186,11 @@ jobs: include-hidden-files: true overwrite: true - Compare-Performance: + ComparePerformance: # This needs to run on Windows to support ScottPlot runs-on: windows-latest - needs: [ Configure, Build-and-Test ] - name: Compare Performance - ${{ inputs.pull-request-id }} + needs: [ Configure, BuildAndTest ] + name: Compare Performance - ${{ inputs.pull-request-id }} steps: - name: Checkout Common uses: actions/checkout@v4 @@ -213,14 +208,13 @@ jobs: merge-multiple: true - name: Compare Performance - shell: pwsh - working-directory: ${{ github.workspace }}/common/ run: | $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) $Options = $(ConvertFrom-Json -AsHashtable '${{ needs.configure.outputs.options }}') - . ${{ github.workspace }}/common/nightly-pr-to-main_compare-performance.ps1 ` + ./nightly-pull-request.compare-performance.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` -GitHubOutput $Env:GITHUB_OUTPUT ` -PullRequestId ${{ inputs.pull-request-id }} ` @@ -239,12 +233,11 @@ jobs: if-no-files-found: ignore include-hidden-files: true - Complete: + Complete: runs-on: ubuntu-latest - name: Complete ${{ inputs.pull-request-id }} - needs: [ Build-and-Test, Compare-Performance ] + name: Complete ${{ inputs.pull-request-id }} + needs: [ BuildAndTest, ComparePerformance ] steps: - - name: Checkout Common uses: actions/checkout@v4 with: @@ -253,15 +246,11 @@ jobs: ref: ${{ inputs.common-ci-ref }} - name: Complete - shell: pwsh - working-directory: ${{ github.workspace }}/common/ run: | $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) - . ${{ github.workspace }}/common/nightly-pr-to-main_complete.ps1 ` + ./nightly-pull-request.complete.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` -GitHubToken ${{ secrets.token }} ` -PullRequestId ${{ inputs.pull-request-id }} ` - -GitHubUser ${{ inputs.user }} ` - -GitHubEmail ${{ inputs.email }} ` -DryRun $DryRun diff --git a/.github/workflows/nightly-prs-to-main.yml b/.github/workflows/nightly-pull-requests.yml similarity index 73% rename from .github/workflows/nightly-prs-to-main.yml rename to .github/workflows/nightly-pull-requests.yml index bdeec13c..cb81778f 100644 --- a/.github/workflows/nightly-prs-to-main.yml +++ b/.github/workflows/nightly-pull-requests.yml @@ -1,36 +1,29 @@ -name: Nightly PRs to Main +name: Nightly Pull Requests on: workflow_call: inputs: repo-name: - required: true type: string + required: true org-name: - required: false type: string - default: 51degrees + default: 51Degrees user: - required: false type: string default: Automation51D email: - required: false type: string - default: "51DCI@51Degrees.com" + default: 51DCI@51degrees.com dryrun: - required: false type: string default: false cache-assets: - default: false type: boolean common-ci-ref: - required: false type: string default: '' prs: # Comma-separated list of pull request IDs to build - required: false type: string secrets: token: @@ -62,11 +55,15 @@ on: UsePublishTests: PearlLicense: +defaults: + run: + shell: pwsh + working-directory: ./common + jobs: # Get all the pull requests that are going to main - Get_Pull_Requests: - outputs: - pull_request_ids: ${{ steps.get_pull_requests.outputs.pull_request_ids }} + GetPullRequests: + name: Get Pull Requests runs-on: ubuntu-latest steps: - name: Checkout Common @@ -79,30 +76,27 @@ jobs: # Get all the pull requests into main, and output the ids - name: Get Pull Requests id: get_pull_requests - shell: pwsh - working-directory: ${{ github.workspace }}/common/ env: PULL_REQUEST_IDS: ${{ inputs.prs }} run: | - . ${{ github.workspace }}/common/nightly-pr-to-main_get-prs.ps1 ` + ./nightly-pull-requests.get-prs.ps1 ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` - -VariableName PullRequestIds ` + -Branch ${{ github.ref_name }} ` -GitHubToken ${{ secrets.token }} ` - -GitHubUser ${{ inputs.user }} ` - -GitHubEmail ${{ inputs.email }} ` - -GitHubOutput $Env:GITHUB_OUTPUT + -GitHubOutput $Env:GITHUB_OUTPUT + outputs: + pull_request_ids: ${{ steps.get_pull_requests.outputs.pull_request_ids }} # Run the common workflow on each pull request - Nightly_PR_to_Main: - needs: Get_Pull_Requests - name: Nightly PR to Main - ${{ matrix.id }} + NightlyPullRequest: + needs: GetPullRequests strategy: fail-fast: false matrix: - id: ${{ fromJSON(needs.Get_Pull_Requests.outputs.pull_request_ids) }} - - uses: ./.github/workflows/nightly-pr-to-main.yml + id: ${{ fromJSON(needs.GetPullRequests.outputs.pull_request_ids) }} + name: Nightly PR - ${{ matrix.id }} + uses: ./.github/workflows/nightly-pull-request.yml with: repo-name: ${{ inputs.repo-name }} org-name: ${{ inputs.org-name }} diff --git a/.github/workflows/nightly-submodule-update.yml b/.github/workflows/nightly-submodule-update.yml index 6503354b..8605dce5 100644 --- a/.github/workflows/nightly-submodule-update.yml +++ b/.github/workflows/nightly-submodule-update.yml @@ -22,34 +22,35 @@ on: required: false type: string default: false + common-ci-ref: + type: string + default: '' secrets: token: required: true jobs: - nightly-submodule-update: + NightlySubmoduleUpdate: runs-on: ubuntu-latest - + name: Nightly Submodule Update steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Checkout reusable workflow dir + - name: Checkout Common uses: actions/checkout@v4 with: repository: ${{ inputs.org-name }}/common-ci path: common - + ref: ${{ inputs.common-ci-ref }} + - name: Update Submodules shell: pwsh working-directory: ${{ github.workspace }}/common/ run: | $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) - . ${{ github.workspace }}/common/nightly-submodule-update.ps1 ` + ./nightly-submodule-update.ps1 ` -GitHubToken ${{ secrets.token }} ` -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` - -RunId ${{ github.run_id }} ` + -Branch ${{ github.ref_name }} ` -GitHubUser ${{ inputs.user }} ` -GitHubEmail ${{ inputs.email }} ` -DryRun $DryRun diff --git a/generate-documentation.ps1 b/generate-documentation.ps1 index 4039a1b8..5a0e9a5f 100644 --- a/generate-documentation.ps1 +++ b/generate-documentation.ps1 @@ -1,79 +1,41 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", + [string]$Branch = "main", + [string]$GitHubUser, + [string]$GitHubEmail, [string]$GitHubToken, - [bool]$SeparateExamples = $False, - [string]$PullRequestId = "" + [bool]$SeparateExamples ) -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - Write-Output "::group::Configure Git" ./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail Write-Output "::endgroup::" Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -if ($PullRequestId -ne 0) { - - ./steps/checkout-pr -RepoName $RepoName -PullRequestId $PullRequestId - -} - if ($SeparateExamples){ $ExamplesRepo = "$RepoName-examples" - + Write-Output "::group::Clone $ExamplesRepo" - ./steps/clone-repo.ps1 -RepoName $ExamplesRepo -OrgName $OrgName -DestinationDir $RepoName + ./steps/clone-repo.ps1 -RepoName $ExamplesRepo -OrgName $OrgName -Branch $Branch -DestinationDir $RepoName/$ExamplesRepo Write-Output "::endgroup::" - - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } } Write-Output "::group::Clone Tools" ./steps/clone-repo.ps1 -RepoName "tools" -OrgName $OrgName Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Clone Documentation" if ($RepoName -ne "documentation") { - ./steps/clone-repo.ps1 -RepoName "documentation" -OrgName $OrgName - - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } + Write-Output "::group::Clone Documentation" + ./steps/clone-repo.ps1 -RepoName "documentation" -OrgName $OrgName -Branch $Branch + Write-Output "::endgroup::" } -Write-Output "::endgroup::" Write-Output "::group::Generate Documentation" ./steps/generate-documentation.ps1 -RepoName $RepoName Write-Output "::endgroup::" - -exit $LASTEXITCODE diff --git a/monthly-copyright-update.ps1 b/monthly-copyright-update.ps1 index efcbff89..cbc912fc 100644 --- a/monthly-copyright-update.ps1 +++ b/monthly-copyright-update.ps1 @@ -1,18 +1,19 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, + [string]$Branch = "main", [Parameter(Mandatory=$true)] [string]$GitHubUser, [Parameter(Mandatory=$true)] [string]$GitHubEmail, [string]$GitHubToken, - [bool]$DryRun = $False + [bool]$DryRun ) +$ErrorActionPreference = "Stop" -. ./constants.ps1 +$PrBranch = "update-copyright/$Branch" # This token is used by the gh command. Write-Output "Setting GITHUB_TOKEN" @@ -22,22 +23,16 @@ Write-Output "::group::Configure Git" ./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail Write-Output "::endgroup::" -Write-Output "::group::Clone $RepoName - $CopyrightUpdateBranch" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $CopyrightUpdateBranch +Write-Output "::group::Clone $RepoName - $Branch" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -ForceSwitchTo $PrBranch Write-Output "::endgroup::" Write-Output "::group::Clone Tools" ./steps/clone-repo.ps1 -RepoName "tools" -OrgName $OrgName Write-Output "::endgroup::" -Write-Output "::group::Options" -$Options = @{ - TargetRepo = $RepoName -} -Write-Output "::endgroup::" - Write-Output "::group::Update Copyright" -./steps/run-repo-script.ps1 -RepoName "tools" -OrgName $OrgName -ScriptName "update-copyright.ps1" -Options $Options -DryRun $DryRun +./steps/run-script.ps1 ./tools/ci/update-copyright.ps1 @{RepoName = tools; TargetRepo = $RepoName} Write-Output "::endgroup::" Write-Output "::group::Has Changed" @@ -45,24 +40,20 @@ Write-Output "::group::Has Changed" Write-Output "::endgroup::" if ($LASTEXITCODE -eq 0) { - Write-Output "::group::Commit Changes" - ./steps/commit-changes.ps1 -RepoName $RepoName -Message "REF: Updated copyright." + ./steps/commit-changes.ps1 -RepoName $RepoName -Message "Update copyright" Write-Output "::endgroup::" Write-Output "::group::Push Changes" - ./steps/push-changes.ps1 -RepoName $RepoName -Branch $CopyrightUpdateBranch -DryRun $DryRun + ./steps/push-changes.ps1 -RepoName $RepoName -DryRun $DryRun -Force $true Write-Output "::endgroup::" Write-Output "::group::PR To Main" - ./steps/pull-request-to-main.ps1 -RepoName $RepoName -Message "Updated copyright." -DryRun $DryRun + ./steps/pull-request.ps1 -RepoName $RepoName -From $PrBranch -To $Branch -Message "Update copyright" -DryRun $DryRun Write-Output "::endgroup::" -} -else { - +} else { Write-Host "No copyright changes, so not creating a pull request." - } -exit 0 +exit # Ignore $LASTEXITCODE here diff --git a/nightly-data-file-change.ps1 b/nightly-data-file-change.ps1 index aac9cd3f..b2d61df2 100644 --- a/nightly-data-file-change.ps1 +++ b/nightly-data-file-change.ps1 @@ -1,97 +1,60 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", + [string]$Branch = "main", + [string]$GitHubUser, + [string]$GitHubEmail, [string]$DeviceDetectionKey, [string]$DeviceDetectionUrl = $Null, [string]$GitHubToken, [bool]$DryRun = $False ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" +$PrBranch = "properties-update/$Branch" Write-Output "::group::Configure Git" ./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail Write-Output "::endgroup::" -Write-Output "::group::Clone $RepoName - $PropertiesUpdateBranch" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $PropertiesUpdateBranch +Write-Output "::group::Clone $RepoName - $Branch" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -ForceSwitchTo $PrBranch Write-Output "::endgroup::" Write-Output "::group::Clone Tools" ./steps/clone-repo.ps1 -RepoName "tools" -OrgName $OrgName Write-Output "::endgroup::" -Write-Output "::group::Options" -$Options = @{ - DeviceDetectionKey = $DeviceDetectionKey - DeviceDetectionUrl = $DeviceDetectionUrl - TargetRepo = $RepoName -} -Write-Output "::endgroup::" - Write-Output "::group::Fetch Assets" -./steps/run-repo-script.ps1 -RepoName "tools" -OrgName $OrgName -ScriptName "fetch-assets.ps1" -Options $Options -DryRun $DryRun +./steps/fetch-hash-assets.ps1 -RepoName tools -LicenseKey $DeviceDetectionKey -Url $DeviceDetectionUrl Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Generate Accessors" -./steps/run-repo-script.ps1 -RepoName "tools" -OrgName $OrgName -ScriptName "generate-accessors.ps1" -Options $Options -DryRun $DryRun +./steps/run-script.ps1 ./tools/ci/generate-accessors.ps1 @{RepoName = tools, TargetRepo = $RepoName} Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Has Changed" ./steps/has-changed.ps1 -RepoName $RepoName Write-Output "::endgroup::" if ($LASTEXITCODE -eq 0) { - Write-Output "::group::Commit Changes" - ./steps/commit-changes.ps1 -RepoName $RepoName -Message "REF: Updated properties." + ./steps/commit-changes.ps1 -RepoName $RepoName -Message "Update properties" Write-Output "::endgroup::" - - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } Write-Output "::group::Push Changes" - ./steps/push-changes.ps1 -RepoName $RepoName -Branch $PropertiesUpdateBranch -DryRun $DryRun + ./steps/push-changes.ps1 -RepoName $RepoName -DryRun $DryRun -Force $true Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Write-Output "::group::PR To Main" - ./steps/pull-request-to-main.ps1 -RepoName $RepoName -Message "Updated properties." -DryRun $DryRun + ./steps/pull-request.ps1 -RepoName $RepoName -From $PrBranch -To $Branch -Message "Update properties" -DryRun $DryRun Write-Output "::endgroup::" -} -else { - +} else { Write-Host "No property changes, so not creating a pull request." - } -exit 0 +exit # Ignore $LASTEXITCODE here diff --git a/nightly-documentation-update.ps1 b/nightly-documentation-update.ps1 index a9922d06..7f6d3c19 100644 --- a/nightly-documentation-update.ps1 +++ b/nightly-documentation-update.ps1 @@ -1,15 +1,15 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", + [string]$GitHubUser, + [string]$GitHubEmail, [string]$GitHubToken, - [bool]$DryRun = $False, - [bool]$SeparateExamples= $False + [bool]$DryRun, + [bool]$SeparateExamples ) +$ErrorActionPreference = "Stop" ./generate-documentation.ps1 ` -RepoName $RepoName ` @@ -19,16 +19,11 @@ param ( -GitHubToken $GitHubToken ` -SeparateExamples $SeparateExamples -if ($SeparateExamples){ - $ExamplesRepo = "$RepoName-examples" - $ExamplesPath = [IO.Path]::Combine($RepoName, $ExamplesRepo) - Write-Output "::group::Removing $ExamplesRepo" - Remove-Item -Path $ExamplesPath -Recurse -Force +if ($SeparateExamples) { + $ExamplesPath = "$RepoName/$RepoName-examples" + Write-Output "::group::Removing $ExamplesPath" + Remove-Item $ExamplesPath -Recurse -Force Write-Output "::endgroup::" - - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } } Write-Output "::group::Has Changed" @@ -36,28 +31,16 @@ Write-Output "::group::Has Changed" Write-Output "::endgroup::" if ($LASTEXITCODE -eq 0) { - Write-Output "::group::Commit Changes" - ./steps/commit-changes.ps1 -RepoName $RepoName -Message "REF: Updated documentation." + ./steps/commit-changes.ps1 -RepoName $RepoName -Message "Update documentation" Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Write-Output "::group::Push Changes" - ./steps/push-changes.ps1 -RepoName $RepoName -Branch gh-pages -DryRun $DryRun + ./steps/push-changes.ps1 -RepoName $RepoName -DryRun $DryRun Write-Output "::endgroup::" - - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } -} -else { - +} else { Write-Host "No property changes, so not pushing changes." - } -exit 0 +exit # Ignore $LASTEXITCODE here diff --git a/nightly-package-update.ps1 b/nightly-package-update.ps1 index 52645971..c5703749 100644 --- a/nightly-package-update.ps1 +++ b/nightly-package-update.ps1 @@ -1,81 +1,50 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, + [string]$Branch = "main", [Parameter(Mandatory=$true)] [string]$GitHubToken, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [string]$RunId = $Null, - [bool]$DryRun = $False + [string]$GitHubUser, + [string]$GitHubEmail, + [bool]$DryRun ) +$ErrorActionPreference = "Stop" -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" +$PrBranch = "update-packages/$Branch" Write-Output "::group::Configure Git" ./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail Write-Output "::endgroup::" Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $PackageUpdateBranch +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -ForceSwitchTo "$PrBranch" Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Update Package Dependencies" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "update-packages.ps1" -DryRun $DryRun +./steps/run-script.ps1 ./$RepoName/ci/update-packages.ps1 @{RepoName = $RepoName; DryRun = $DryRun} Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Check for Changes" ./steps/has-changed.ps1 -RepoName $RepoName Write-Output "::endgroup::" if ($LASTEXITCODE -eq 0) { - Write-Output "::group::Commit Changes" - ./steps/commit-changes.ps1 -RepoName $RepoName -Message "REF: Updated packages." + ./steps/commit-changes.ps1 -RepoName $RepoName -Message "Update packages" Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Write-Output "::group::Push Changes" - ./steps/push-changes.ps1 -RepoName $RepoName -Branch $PackageUpdateBranch -DryRun $DryRun + ./steps/push-changes.ps1 -RepoName $RepoName -DryRun $DryRun -Force $true Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Write-Output "::group::Create Pull Request" - ./steps/pull-request-to-main.ps1 -RepoName $RepoName -Message "Updated packages." -DryRun $DryRun + ./steps/pull-request.ps1 -RepoName $RepoName -From $PrBranch -To $Branch -Message "Update packages" -DryRun $DryRun Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - } else { Write-Output "$($env:CI -eq 'true' ? '::warning::' : '')No package changes, so not creating a pull request." - exit 0 } + +exit # Ignore $LASTEXITCODE here diff --git a/nightly-pr-to-main_build-and-test.ps1 b/nightly-pr-to-main_build-and-test.ps1 deleted file mode 100644 index b962a55a..00000000 --- a/nightly-pr-to-main_build-and-test.ps1 +++ /dev/null @@ -1,105 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [string]$GitHubOutput, - [Parameter(Mandatory=$true)] - [string]$PullRequestId, - [Parameter(Mandatory=$true)] - [Hashtable]$Options, - [bool]$DryRun = $False -) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $BranchName -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Checkout PR" -./steps/checkout-pr.ps1 -RepoName $RepoName -PullRequestId $PullRequestId -VariableName "PullRequestSha" -Write-Output pr-sha="$PullRequestSha" | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Fetch Assets" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "fetch-assets.ps1" -Options ($Options.Keys ?? @{}) -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Setup Environment" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "setup-environment.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Build Project" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "build-project.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Run Unit Tests" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "run-unit-tests.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Run Integration Tests" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "run-integration-tests.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -if ($Options.RunPerformance -eq $True) { - Write-Output "::group::Run Performance Tests" - ./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "run-performance-tests.ps1" -Options $Options -DryRun $DryRun - Write-Output "::endgroup::" - - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } -} -else { - Write-Output "Skipping performance tests as they are not configured for '$($Options.Name)'" -} diff --git a/nightly-pr-to-main_complete.ps1 b/nightly-pr-to-main_complete.ps1 deleted file mode 100644 index 58a7bc8c..00000000 --- a/nightly-pr-to-main_complete.ps1 +++ /dev/null @@ -1,47 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [string]$PullRequestId, - [bool]$DryRun = $False -) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $BranchName -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Complete Pull Request" -./steps/merge-pr.ps1 -RepoName $RepoName -OrgName $OrgName -PullRequestId $PullRequestId -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-pr-to-main_configure-pr.ps1 b/nightly-pr-to-main_configure-pr.ps1 deleted file mode 100644 index a64f6fb4..00000000 --- a/nightly-pr-to-main_configure-pr.ps1 +++ /dev/null @@ -1,60 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [Parameter(Mandatory=$true)] - [string]$GitHubOutput, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [string]$PullRequestId -) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $BranchName -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Checkout PR" -./steps/checkout-pr.ps1 -RepoName $RepoName -PullRequestId $PullRequestId -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Get Build Options" -$OptionsFile = [IO.Path]::Combine($pwd, $RepoName, "ci", "options.json") -$Options = Get-Content $OptionsFile -Raw -$Options = $Options -replace "`r`n", "" -replace "`n", "" -Write-Output $Options -Write-Output options=$Options | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-pr-to-main_get-prs.ps1 b/nightly-pr-to-main_get-prs.ps1 deleted file mode 100644 index b938226b..00000000 --- a/nightly-pr-to-main_get-prs.ps1 +++ /dev/null @@ -1,57 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [string]$GitHubOutput, - [Parameter(Mandatory=$true)] - [string]$VariableName -) - -# If the list of pull requests is provided as a CI parameter - output it and exit early. -# An environment variable is used instead of a parameter to avoid arbitrary code injection. -if ($env:PULL_REQUEST_IDS) { - Write-Output "Pull request ids are: $env:PULL_REQUEST_IDS" - "pull_request_ids=[$env:PULL_REQUEST_IDS]" | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append - return -} - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $BranchName -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Get Pull Requests" -./steps/get-pull-requests.ps1 -RepoName $RepoName -OrgName $OrgName -VariableName $VariableName -GitHubToken $GitHubToken -Write-Output pull_request_ids="[$([string]::Join(",", $PullRequestIds))]" | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-publish-main_build.ps1 b/nightly-publish-main_build.ps1 deleted file mode 100644 index cf5d7b71..00000000 --- a/nightly-publish-main_build.ps1 +++ /dev/null @@ -1,48 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [string]$Branch = "", - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [Hashtable]$Options, - [bool]$DryRun = $False -) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Build Package" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "build-package.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-publish-main_configure.ps1 b/nightly-publish-main_configure.ps1 deleted file mode 100644 index 6fd48425..00000000 --- a/nightly-publish-main_configure.ps1 +++ /dev/null @@ -1,104 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [string]$Branch = "", - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [string]$GitHubOutput, - [bool]$DryRun = $False, - [string]$BuildPlatform -) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Get Next Package Version" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "get-next-package-version.ps1" -Options @{VariableName = "Version"} -DryRun $DryRun -Write-Output version=$Version -$IsValid = [bool]([regex]::Match($Version, '^(\d+)\.(\d+)\.(\d+)(-SNAPSHOT)?$')) -if ($IsValid -eq $False) { - Write-Error "The package version was not valid: '$Version'" - exit 1 -} -Write-Output version=$Version | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Package Update Required" -try { - ./steps/package-update-required.ps1 -RepoName $RepoName -Version $Version -} finally { - if ($LASTEXITCODE -eq 0) { - Write-Output update_required=true | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append - } else { - Write-Output update_required=false | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append - # Set a zero exit code as we don't want to fail just because an update is not required. - $LASTEXITCODE = 0 - } -} -Write-Output "::endgroup::" - -Write-Output "::group::Get Options" -$OptionsFile = [IO.Path]::Combine($pwd, $RepoName, "ci", "options.json") - -$Options = Get-Content $OptionsFile | ConvertFrom-Json - -Write-Output $Options - -$RequiredOptions = @() -foreach ($element in $Options) -{ - if ($element.PackageRequirement) { - $element | Add-Member -Name "Version" -Value $Version -MemberType NoteProperty - $RequiredOptions += $element - } -} -if ($RequiredOptions.Count -eq 0) { - # As there are no prebuild steps, add an empty one to ensure the YAML is still valid. - $RequiredOptions += @{ - "Name" = "No Prebuild" - "PackageRequirement" = $False - "Image" = $BuildPlatform - } -} -$RequiredOptions = $RequiredOptions | ConvertTo-Json -AsArray -$RequiredOptions = $RequiredOptions -replace "`r`n", "" -replace "`n", "" -Write-Host $RequiredOptions -Write-Output options=$RequiredOptions | Out-File -FilePath $GitHubOutput -Encoding utf8 -Append -Write-Output $RequiredOptions -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-publish-main_prebuild.ps1 b/nightly-publish-main_prebuild.ps1 deleted file mode 100644 index e703afbc..00000000 --- a/nightly-publish-main_prebuild.ps1 +++ /dev/null @@ -1,64 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [string]$Branch = "", - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [Hashtable]$Options, - [bool]$DryRun = $False -) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Fetch Assets" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "fetch-assets.ps1" -Options $Options.Keys -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Setup Environment" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "setup-environment.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Build Package Requirements" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "build-package-requirements.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-publish-main_test.ps1 b/nightly-publish-main_test.ps1 deleted file mode 100644 index 6e72a2e9..00000000 --- a/nightly-publish-main_test.ps1 +++ /dev/null @@ -1,73 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$GitHubToken, - [string]$Branch = "", - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [Parameter(Mandatory=$true)] - [Hashtable]$Options, - [bool]$DryRun = $False -) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -Write-Output "::group::Configure Git" -./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail -Write-Output "::endgroup::" - -Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Fetch Assets" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "fetch-assets.ps1" -Options $Options.Keys -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Setup Environment" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "setup-environment.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -Write-Output "::group::Install Package From Artifact" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "install-package.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - - -Write-Output "::group::Run Integration Tests" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "run-integration-tests.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-publish.build.ps1 b/nightly-publish.build.ps1 new file mode 100644 index 00000000..dc0d72fd --- /dev/null +++ b/nightly-publish.build.ps1 @@ -0,0 +1,32 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [string]$Branch = "main", + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [string]$GitHubUser, + [string]$GitHubEmail, + [Parameter(Mandatory=$true)] + [hashtable]$Options, + [bool]$DryRun +) +$ErrorActionPreference = "Stop" + +$Options += $PSBoundParameters # Add RepoName, DryRun etc. +if ($Options.Keys) { + $Options += $Options.Keys # Expand keys into options +} + +Write-Output "::group::Configure Git" +./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail +Write-Output "::endgroup::" + +Write-Output "::group::Clone $RepoName" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch +Write-Output "::endgroup::" + +Write-Output "::group::Build Package" +./steps/run-script.ps1 ./$RepoName/ci/build-package.ps1 $Options +Write-Output "::endgroup::" diff --git a/nightly-publish.configure.ps1 b/nightly-publish.configure.ps1 new file mode 100644 index 00000000..e9b44a4a --- /dev/null +++ b/nightly-publish.configure.ps1 @@ -0,0 +1,67 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [string]$Branch = "main", + [string]$GitHubUser, + [string]$GitHubEmail, + [Parameter(Mandatory=$true)] + [string]$GitHubOutput, + [bool]$DryRun, + [string]$BuildPlatform = "ubuntu-latest" +) +$ErrorActionPreference = "Stop" + +Write-Output "::group::Configure Git" +./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail +Write-Output "::endgroup::" + +Write-Output "::group::Clone $RepoName" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch +Write-Output "::endgroup::" + +Write-Output "::group::Get Next Package Version" +./steps/run-script.ps1 ./$RepoName/ci/get-next-package-version.ps1 @{RepoName = $RepoName; VariableName = "Version"} +Write-Output version=$Version +if (!($Version -match '^(\d+)\.(\d+)\.(\d+)(\.\d+)?(-SNAPSHOT)?$')) { + Write-Error "Version '$Version' isn't valid" + exit 1 +} +Write-Output version=$Version | Out-File $GitHubOutput -Append +Write-Output "::endgroup::" + +Write-Output "::group::Package Update Required" +./steps/package-update-required.ps1 -RepoName $RepoName -Version $Version +Write-Output "update_required=$($LASTEXITCODE -eq 0 ? 'true' : 'false')" | Out-File $GitHubOutput -Append +Write-Output "::endgroup::" + +Write-Output "::group::Get Options" +$Options = Get-Content $RepoName/ci/options.json | ConvertFrom-Json +Write-Output $Options + +$RequiredOptions = @() +foreach ($element in $Options) { + if ($element.PackageRequirement) { + $element | Add-Member -Name "Version" -Value $Version -MemberType NoteProperty + $RequiredOptions += $element + } +} +if ($RequiredOptions.Count -eq 0) { + # As there are no prebuild steps, add an empty one to ensure the YAML is still valid. + $RequiredOptions += @{ + "Name" = "No Prebuild" + "PackageRequirement" = $False + "Image" = $BuildPlatform + } +} +$RequiredOptions = $RequiredOptions | ConvertTo-Json -AsArray +$RequiredOptions = $RequiredOptions -replace "`r?`n", "" +Write-Host $RequiredOptions +Write-Output options=$RequiredOptions | Out-File $GitHubOutput -Append +Write-Output $RequiredOptions +Write-Output "::endgroup::" + +exit # Ignore $LASTEXITCODE here diff --git a/nightly-publish-main_package.ps1 b/nightly-publish.package.ps1 similarity index 57% rename from nightly-publish-main_package.ps1 rename to nightly-publish.package.ps1 index 5941b72e..cdffaade 100644 --- a/nightly-publish-main_package.ps1 +++ b/nightly-publish.package.ps1 @@ -1,31 +1,24 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, + [string]$Branch = "main", [Parameter(Mandatory=$true)] [string]$GitHubToken, - [string]$Branch = "", - [string]$GitHubUser = "", - [string]$GitHubEmail = "", + [string]$GitHubUser, + [string]$GitHubEmail, [Parameter(Mandatory=$true)] - [Hashtable]$Options, + [hashtable]$Options, [bool]$DryRun = $False ) +$ErrorActionPreference = "Stop" -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser +# Common options +$Options += $PSBoundParameters # Add RepoName, DryRun etc. +if ($Options.Keys) { + $Options += $Options.Keys # Expand keys into options } -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" Write-Output "::group::Configure Git" ./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail @@ -35,25 +28,17 @@ Write-Output "::group::Clone $RepoName" ./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Install Package From Artifact" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "install-package.ps1" -Options $Options -DryRun $DryRun +./steps/run-script.ps1 ./$RepoName/ci/install-package.ps1 $Options Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Publish Packages" -./steps/run-repo-script.ps1 -RepoName $RepoName -OrgName $OrgName -ScriptName "publish-package.ps1" -Options $Options -DryRun $DryRun -Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE +if ($Branch -eq "main") { + ./steps/run-script.ps1 ./$RepoName/ci/publish-package.ps1 $Options +} else { + Write-Output "Not on the main branch, skipping publishing" } +Write-Output "::endgroup::" Write-Output "::group::Update Tag" if ($global:SkipUpdateTag) { # Using a global here so that it can be set by publish-package.ps1 @@ -63,7 +48,3 @@ if ($global:SkipUpdateTag) { # Using a global here so that it can be set by publ ./steps/upload-release-assets.ps1 -RepoName $RepoName -OrgName $OrgName -Tag $Options.Version -DryRun $DryRun } Write-Output "::endgroup::" - -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} diff --git a/nightly-publish.prebuild.ps1 b/nightly-publish.prebuild.ps1 new file mode 100644 index 00000000..56a56d68 --- /dev/null +++ b/nightly-publish.prebuild.ps1 @@ -0,0 +1,40 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [string]$Branch = "main", + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [string]$GitHubUser, + [string]$GitHubEmail, + [Parameter(Mandatory=$true)] + [Hashtable]$Options, + [bool]$DryRun +) +$ErrorActionPreference = "Stop" + +$Options += $PSBoundParameters # Add RepoName, DryRun etc. +if ($Options.Keys) { + $Options += $Options.Keys # Expand keys into options +} + +Write-Output "::group::Configure Git" +./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail +Write-Output "::endgroup::" + +Write-Output "::group::Clone $RepoName" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch +Write-Output "::endgroup::" + +Write-Output "::group::Fetch Assets" +./steps/run-script.ps1 ./$RepoName/ci/fetch-assets.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Setup Environment" +./steps/run-script.ps1 ./$RepoName/ci/setup-environment.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Build Package Requirements" +./steps/run-script.ps1 ./$RepoName/ci/build-package-requirements.ps1 $Options +Write-Output "::endgroup::" diff --git a/nightly-publish.test.ps1 b/nightly-publish.test.ps1 new file mode 100644 index 00000000..c08668a2 --- /dev/null +++ b/nightly-publish.test.ps1 @@ -0,0 +1,44 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [string]$Branch = "main", + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [string]$GitHubUser = "", + [string]$GitHubEmail = "", + [Parameter(Mandatory=$true)] + [Hashtable]$Options, + [bool]$DryRun = $False +) +$ErrorActionPreference = "Stop" + +$Options += $PSBoundParameters # Add RepoName, DryRun etc. +if ($Options.Keys) { + $Options += $Options.Keys # Expand keys into options +} + +Write-Output "::group::Configure Git" +./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail +Write-Output "::endgroup::" + +Write-Output "::group::Clone $RepoName" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch +Write-Output "::endgroup::" + +Write-Output "::group::Fetch Assets" +./steps/run-script.ps1 ./$RepoName/ci/fetch-assets.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Setup Environment" +./steps/run-script.ps1 ./$RepoName/ci/setup-environment.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Install Package From Artifact" +./steps/run-script.ps1 ./$RepoName/ci/install-package.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Run Integration Tests" +./steps/run-script.ps1 ./$RepoName/ci/run-integration-tests.ps1 $Options +Write-Output "::endgroup::" diff --git a/nightly-pull-request.build-and-test.ps1 b/nightly-pull-request.build-and-test.ps1 new file mode 100644 index 00000000..582d4668 --- /dev/null +++ b/nightly-pull-request.build-and-test.ps1 @@ -0,0 +1,66 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [string]$Branch = "main", + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [string]$GitHubUser, + [string]$GitHubEmail, + [Parameter(Mandatory=$true)] + [string]$GitHubOutput, + [Parameter(Mandatory=$true)] + [string]$PullRequestId, + [Parameter(Mandatory=$true)] + [hashtable]$Options, + [bool]$DryRun +) +$ErrorActionPreference = "Stop" + +$Options += $PSBoundParameters # Add RepoName, DryRun etc. +if ($Options.Keys) { + $Options += $Options.Keys # Expand keys into options +} + +Write-Output "::group::Configure Git" +./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail +Write-Output "::endgroup::" + +Write-Output "::group::Clone $RepoName" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch +Write-Output "::endgroup::" + +Write-Output "::group::Checkout PR" +./steps/checkout-pr.ps1 -RepoName $RepoName -Branch $Branch -PullRequestId $PullRequestId -SetVariable PullRequestSha +Write-Output "pr-sha=$PullRequestSha" | Out-File $GitHubOutput -Append +Write-Output "::endgroup::" + +Write-Output "::group::Fetch Assets" +./steps/run-script.ps1 ./$RepoName/ci/fetch-assets.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Setup Environment" +./steps/run-script.ps1 ./$RepoName/ci/setup-environment.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Build Project" +./steps/run-script.ps1 ./$RepoName/ci/build-project.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Run Unit Tests" +./steps/run-script.ps1 ./$RepoName/ci/run-unit-tests.ps1 $Options +Write-Output "::endgroup::" + +Write-Output "::group::Run Integration Tests" +./steps/run-script.ps1 ./$RepoName/ci/run-integration-tests.ps1 $Options +Write-Output "::endgroup::" + +if ($Options.RunPerformance -eq $True) { + Write-Output "::group::Run Performance Tests" + ./steps/run-script.ps1 ./$RepoName/ci/run-performance-tests.ps1 $Options + Write-Output "::endgroup::" +} +else { + Write-Output "Skipping performance tests as they are not configured for '$($Options.Name)'" +} diff --git a/nightly-pr-to-main_compare-performance.ps1 b/nightly-pull-request.compare-performance.ps1 similarity index 72% rename from nightly-pr-to-main_compare-performance.ps1 rename to nightly-pull-request.compare-performance.ps1 index 27974267..abc5a887 100644 --- a/nightly-pr-to-main_compare-performance.ps1 +++ b/nightly-pull-request.compare-performance.ps1 @@ -1,49 +1,33 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, + [string]$Branch = "main", [Parameter(Mandatory=$true)] [string]$GitHubToken, [Parameter(Mandatory=$true)] [string]$GitHubOutput, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", + [string]$GitHubUser, + [string]$GitHubEmail, [Parameter(Mandatory=$true)] [string]$PullRequestId, [Parameter(Mandatory=$true)] - $Options, - $RunId = 0, - [bool]$DryRun = $False + [hashtable[]]$Options, + [string]$RunId = '0', + [bool]$DryRun ) - -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" +$ErrorActionPreference = "Stop" Write-Output "::group::Configure Git" ./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail Write-Output "::endgroup::" Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $BranchName +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - -if ($True -ne $env:CI) { +if ($env:CI -ne "true") { # The REST API cannot be used while the workflow is running. So in CI, # a GitHub Actions step is used instead. # See https://github.com/actions/upload-artifact/issues/53 @@ -52,9 +36,6 @@ if ($True -ne $env:CI) { Write-Output "::endgroup::" } - Write-Output "::group::Compare Performance Results" ./steps/compare-performance.ps1 -RepoName $RepoName -OrgName $OrgName -RunId $RunId -PullRequestId $PullRequestId -AllOptions $Options -DryRun $DryRun Write-Output "::endgroup::" - -exit $LASTEXITCODE diff --git a/nightly-pull-request.complete.ps1 b/nightly-pull-request.complete.ps1 new file mode 100644 index 00000000..65adac2f --- /dev/null +++ b/nightly-pull-request.complete.ps1 @@ -0,0 +1,18 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [Parameter(Mandatory=$true)] + [string]$PullRequestId, + [bool]$DryRun +) +$ErrorActionPreference = "Stop" + +# This token is used by the gh command. +Write-Output "Setting GITHUB_TOKEN" +$env:GITHUB_TOKEN="$GitHubToken" + +./steps/merge-pr.ps1 -RepoName $RepoName -OrgName $OrgName -PullRequestId $PullRequestId -DryRun $DryRun diff --git a/nightly-pull-request.configure-pr.ps1 b/nightly-pull-request.configure-pr.ps1 new file mode 100644 index 00000000..fee6a335 --- /dev/null +++ b/nightly-pull-request.configure-pr.ps1 @@ -0,0 +1,35 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [string]$Branch = "main", + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [Parameter(Mandatory=$true)] + [string]$GitHubOutput, + [string]$GitHubUser, + [string]$GitHubEmail, + [Parameter(Mandatory=$true)] + [string]$PullRequestId +) +$ErrorActionPreference = "Stop" + +Write-Output "::group::Configure Git" +./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail +Write-Output "::endgroup::" + +Write-Output "::group::Clone $RepoName" +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch +Write-Output "::endgroup::" + +Write-Output "::group::Checkout PR" +./steps/checkout-pr.ps1 -RepoName $RepoName -Branch $Branch -PullRequestId $PullRequestId +Write-Output "::endgroup::" + +Write-Output "::group::Get Build Options" +$Options = Get-Content $RepoName/ci/options.json -Raw +$Options = $Options -replace "`r?`n", "" +Write-Output $Options +Write-Output options=$Options | Out-File $GitHubOutput -Append +Write-Output "::endgroup::" diff --git a/nightly-pull-requests.get-prs.ps1 b/nightly-pull-requests.get-prs.ps1 new file mode 100644 index 00000000..7c70b7ec --- /dev/null +++ b/nightly-pull-requests.get-prs.ps1 @@ -0,0 +1,29 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$OrgName, + [string]$Branch = "main", + [Parameter(Mandatory=$true)] + [string]$GitHubToken, + [Parameter(Mandatory=$true)] + [string]$GitHubOutput +) +$ErrorActionPreference = "Stop" + +# If the list of pull requests is provided as a CI parameter - output it and exit early. +# An environment variable is used instead of a parameter to avoid arbitrary code injection. +if ($env:PULL_REQUEST_IDS) { + Write-Output "Pull request ids are: $env:PULL_REQUEST_IDS" + "pull_request_ids=[$env:PULL_REQUEST_IDS]" | Out-File $GitHubOutput -Append + exit +} + +# This token is used by the gh command. +Write-Output "Setting GITHUB_TOKEN" +$env:GITHUB_TOKEN="$GitHubToken" + +Write-Output "::group::Get Pull Requests" +./steps/get-pull-requests.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -SetVariable PullRequestIds +Write-Output "pull_request_ids=[$($PullRequestIds -Join ',')]" | Out-File $GitHubOutput -Append +Write-Output "::endgroup::" diff --git a/nightly-submodule-update.ps1 b/nightly-submodule-update.ps1 index ba4b596b..b28f6961 100644 --- a/nightly-submodule-update.ps1 +++ b/nightly-submodule-update.ps1 @@ -1,81 +1,51 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, + [string]$Branch = "main", [Parameter(Mandatory=$true)] [string]$GitHubToken, - [string]$GitHubUser = "", - [string]$GitHubEmail = "", - [string]$RunId = $Null, + [string]$GitHubUser, + [string]$GitHubEmail, [bool]$DryRun = $False ) +$ErrorActionPreference = "Stop" -. ./constants.ps1 - -if ($GitHubUser -eq "") { - $GitHubUser = $DefaultGitUser -} -if ($GitHubEmail -eq "") { - $GitHubEmail = $DefaultGitEmail -} - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" +$PrBranch = "update-submodules/$Branch" Write-Output "::group::Configure Git" ./steps/configure-git.ps1 -GitHubToken $GitHubToken -GitHubUser $GitHubUser -GitHubEmail $GitHubEmail Write-Output "::endgroup::" Write-Output "::group::Clone $RepoName" -./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $SubModuleUpdateBranch +./steps/clone-repo.ps1 -RepoName $RepoName -OrgName $OrgName -Branch $Branch -ForceSwitchTo $PrBranch Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Update Submodules" ./steps/update-sub-modules.ps1 -RepoName $RepoName Write-Output "::endgroup::" -if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE -} - Write-Output "::group::Check for Changes" ./steps/has-changed.ps1 -RepoName $RepoName Write-Output "::endgroup::" if ($LASTEXITCODE -eq 0) { - Write-Output "::group::Commit Changes" - ./steps/commit-changes.ps1 -RepoName $RepoName -Message "REF: Updated submodules." + ./steps/commit-changes.ps1 -RepoName $RepoName -Message "Update submodules" Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Write-Output "::group::Push Changes" - ./steps/push-changes.ps1 -RepoName $RepoName -Branch $SubModuleUpdateBranch -DryRun $DryRun + git config user.name + ./steps/push-changes.ps1 -RepoName $RepoName -DryRun $DryRun -Force $true Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - Write-Output "::group::Create Pull Request" - ./steps/pull-request-to-main.ps1 -RepoName $RepoName -Message "Updated submodule." -DryRun $DryRun + ./steps/pull-request.ps1 -RepoName $RepoName -From $PrBranch -To $Branch -Message "Update submodules" -DryRun $DryRun Write-Output "::endgroup::" - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - } else { Write-Output "$($env:CI -eq 'true' ? '::warning::' : '')No submodule changes, so not creating a pull request." - exit 0 } + +exit # Ignore $LASTEXITCODE here diff --git a/python/run-integration-tests.ps1 b/python/run-integration-tests.ps1 index 4ccc31c6..1ece0258 100644 --- a/python/run-integration-tests.ps1 +++ b/python/run-integration-tests.ps1 @@ -47,7 +47,7 @@ try { # coverage run -m xmlrunner discover -s tests -p 'test*.py' -o $commonTestResults || $($testsFailed = $true) $packageName = Split-Path -Path $pwd -Leaf # Required for location-python, which uses . as package path $coverageOutputFile = Join-Path -Path $commonTestResults -ChildPath "$packageName.xml" - $toxEnv = $env:GITHUB_JOB -eq "Build-and-Test" ? "py" : "pre-publish" + $toxEnv = $env:GITHUB_JOB -ilike "build*test" ? "py" : "pre-publish" python -m tox -e $toxEnv -- --junit-xml=$coverageOutputFile || $($testsFailed = $true) Move-Item -Path .coverage -Destination $repoPath/.coverage.$packageName || $(throw "failed to move coverage report") } finally { diff --git a/python/run-unit-tests.ps1 b/python/run-unit-tests.ps1 index 02cab399..2406b09a 100644 --- a/python/run-unit-tests.ps1 +++ b/python/run-unit-tests.ps1 @@ -18,7 +18,7 @@ try { Write-Output "Running tests in '$pwd'" # coverage run -m xmlrunner discover -s tests -p 'test*.py' -o $commonTestResults || $($testsFailed = $true) $coverageOutputFile = Join-Path -Path $commonTestResults -ChildPath "$package.xml" - $toxEnv = $env:GITHUB_JOB -eq "Build-and-Test" ? "py" : "pre-publish" + $toxEnv = $env:GITHUB_JOB -ilike "build*test" ? "py" : "pre-publish" python -m tox -e $toxEnv -- --junit-xml=$coverageOutputFile || $($testsFailed = $true) Move-Item -Path .coverage -Destination $repoPath/.coverage.$package || $(throw "failed to move coverage report") } finally { diff --git a/steps/checkout-pr.ps1 b/steps/checkout-pr.ps1 index 5d44ebec..adfff6ff 100644 --- a/steps/checkout-pr.ps1 +++ b/steps/checkout-pr.ps1 @@ -3,53 +3,33 @@ param ( [string]$RepoName, [Parameter(Mandatory=$true)] [int]$PullRequestId, - [string]$VariableName = "PullRequestSha" + [string]$Branch = "main", + [string]$SetVariable = "PullRequestSha" ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -. ./constants.ps1 - -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - +Write-Output "Entering $RepoName" +Push-Location $RepoName try { - if ($PullRequestId -eq 0) { - Write-Output "Not running for a PR" - exit 0 - + exit } $PrTitle = gh pr view $PullRequestId --json number,headRefName,baseRefName,title -t '#{{.number}} {{.headRefName}}->{{.baseRefName}} : {{.title}}' - Write-Output "Checking out PR $PrTitle" - gh pr checkout $PullRequestId - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } + Write-Output "Checking out PR '$PrTitle'" + gh pr checkout --force --recurse-submodules $PullRequestId - Write-Output "Merging in any changes from main" - git merge origin/main - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - - # Any submodules may not have updated, so do this manually. - git submodule update --init --recursive - if ($LASTEXITCODE -ne 0) { - exit $LASTEXITCODE - } - - $Sha = gh pr view $PullRequestId --json headRefOid --jq '.headRefOid' - Write-Output "Setting '$VariableName' to '$Sha'" - Set-Variable -Name $VariableName -Value $Sha -Scope Global + Write-Output "Merging in any changes from $Branch" + git merge origin/$Branch -} -finally { + $Sha = gh pr view $PullRequestId --json headRefOid --jq '.headRefOid' + Write-Output "Setting '$SetVariable' to '$Sha'" + Set-Variable -Scope 1 -Name $SetVariable -Value $Sha - Write-Output "Leaving '$RepoPath'" +} finally { + Write-Output "Leaving $RepoName" Pop-Location - } diff --git a/steps/clone-repo.ps1 b/steps/clone-repo.ps1 index 0f46664a..974acfdb 100644 --- a/steps/clone-repo.ps1 +++ b/steps/clone-repo.ps1 @@ -3,53 +3,32 @@ param ( [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, - [string]$Branch, - [string]$DestinationDir = "" + [string]$Branch = "main", + [string]$ForceSwitchTo, + [string]$DestinationDir = "." ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -. ./constants.ps1 +# Using short directory name to avoid/delay problems with long paths when cloning on Windows +$tmp = "$DestinationDir/b" -$TemporaryRepoName = [IO.Path]::Combine($DestinationDir, "b") -$Url = "https://github.com/$OrgName/$RepoName" -$RepoPath = [IO.Path]::Combine($pwd, $TemporaryRepoName) +git clone --recurse-submodules --shallow-submodules --branch $Branch "https://github.com/$OrgName/$RepoName" $tmp -Write-Output "Cloning '$Url'" -git clone $Url $TemporaryRepoName - - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - if ("" -ne $Branch) { - # The format %(refname) returns the branches in the format "refs/remotes/[remotename]/[branchname]" - $branches = $(git branch -a --format "%(refname)") - - if ($branches.Contains("refs/remotes/origin/$Branch")) { - - Write-Output "Checking out branch '$Branch'" - git checkout $Branch +if ($ForceSwitchTo) { + Write-Output "Switching to $ForceSwitchTo" + git -C b switch --recurse-submodules -C $ForceSwitchTo +} - } - else { +git -C b log -1 - Write-Output "Creating new branch '$Branch'" - git checkout -b $Branch +Write-Output "Renaming '$tmp' to '$RepoName'" +Rename-Item $tmp $RepoName - } +<#if ($Branch) { + & { + $PSNativeCommandUseErrorActionPreference = $false + git -C $RepoName show-ref --quiet --branches $Branch } - - Write-Output "Checking out submodules" - git submodule update --init --recursive - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location - - Write-Output "Rename temporary directory" - Rename-Item $RepoPath $RepoName - -} + git switch ($LASTEXITCODE -ne 0 ? '-c' : $null) $Branch +}#> diff --git a/steps/commit-changes.ps1 b/steps/commit-changes.ps1 index f8a26ad2..6f6e5715 100644 --- a/steps/commit-changes.ps1 +++ b/steps/commit-changes.ps1 @@ -1,28 +1,14 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [Parameter(Mandatory=$true)] [string]$Message ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - Write-Output "Adding '$($(git diff -s).count)' changes" - git add * - - Write-Output "Committing changes with message '$Message'" - git commit -m $Message - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location +Write-Output "Adding $($(git -C $RepoName status --porcelain).Count) changes" +git -C $RepoName add . -} \ No newline at end of file +Write-Output "Committing changes with message '$Message'" +git -C $RepoName commit -m $Message diff --git a/steps/configure-git.ps1 b/steps/configure-git.ps1 index 93a36956..75c6a041 100644 --- a/steps/configure-git.ps1 +++ b/steps/configure-git.ps1 @@ -1,14 +1,22 @@ param ( [Parameter(Mandatory=$true)] $GitHubToken, - [Parameter(Mandatory=$true)] $GitHubUser, - [Parameter(Mandatory=$true)] $GitHubEmail ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true + +$GitHubUser = $GitHubUser ? $GitHubUser : "Automation51D" +$GitHubEmail = $GitHubEmail ? $GitHubEmail : "CI@51Degrees.com" + git config --global url.https://$GitHubToken@github.com/.insteadOf https://github.com/ git config --global user.email $GitHubEmail git config --global user.name $GitHubUser git config --global filter.lfs.smudge "git-lfs smudge --skip -- %f" git config --global filter.lfs.process "git-lfs filter-process --skip" -git config --global core.longpaths true \ No newline at end of file +git config --global core.longpaths true + +# This token is used by the gh command. +Write-Output "Setting GITHUB_TOKEN" +$env:GITHUB_TOKEN="$GitHubToken" diff --git a/steps/fetch-hash-assets.ps1 b/steps/fetch-hash-assets.ps1 index 9e180385..e3cd743b 100644 --- a/steps/fetch-hash-assets.ps1 +++ b/steps/fetch-hash-assets.ps1 @@ -38,4 +38,4 @@ finally { Write-Output "Leaving '$RepoPath'" Pop-Location -} \ No newline at end of file +} diff --git a/steps/generate-documentation.ps1 b/steps/generate-documentation.ps1 index 5c07981a..e6a5523c 100644 --- a/steps/generate-documentation.ps1 +++ b/steps/generate-documentation.ps1 @@ -1,11 +1,13 @@ param ( + [Parameter(Mandatory=$true)] [string]$RepoName ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) -$DocsPath = [IO.Path]::Combine($RepoPath, "docs") -$DoxyGenPath = [IO.Path]::Combine($pwd, "tools", "DoxyGen") -$DoxyGen = [IO.Path]::Combine($DoxyGenPath, "doxygen") +$DocsPath = "$PWD/$RepoName/docs" +$DoxyGenPath = "$PWD/tools/DoxyGen" +$DoxyGen = "$DoxyGenPath/doxygen" Write-Output "Setting up requirements" sudo apt-get install -y graphviz flex bison @@ -13,13 +15,11 @@ sudo apt-get install -y graphviz flex bison Write-Output "Entering '$DoxyGenPath'" Push-Location $DoxyGenPath try { - Write-Output "Extracting DoxyGen executable" unzip -o doxygen-linux.zip Move-Item doxygen-linux $DoxyGen -Force -} -finally { +} finally { Write-Output "Leaving '$DoxyGenPath'" Pop-Location } @@ -30,51 +30,45 @@ chmod +x $DoxyGen Write-Output "Entering '$DocsPath'" Push-Location $DocsPath try { - Write-Output "Running DoxyGen" & $DoxyGen -} -finally { - +} finally { Write-Output "Leaving '$DocsPath'" Pop-Location - } -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath +Write-Output "Entering '$RepoName'" +Push-Location $RepoName try { - $VersionPath = Get-ChildItem -Path $pwd -Filter "4.*" + $VersionPath = Get-ChildItem -Filter "4.*" Write-Output "Moving $($VersionPath.FullName)" Move-Item $VersionPath.FullName "$($VersionPath.FullName)-new" - # Check out the gh-pages branch so we're ready to commit images. - $branches = $(git branch -a --format "%(refname)") - $PagesBranch = "gh-pages" - if ($branches.Contains("refs/remotes/origin/$PagesBranch")) { - Write-Output "Checking out branch '$PagesBranch'" - git checkout $PagesBranch -f --recurse-submodules - } - else { - Write-Output "Creating new branch '$PagesBranch'" - git checkout --orphan $PagesBranch - git rm -rf . + $branch = "gh-pages" + Write-Output "Switching to branch '$branch'" + & { + $PSNativeCommandUseErrorActionPreference = $false + git show-ref --quiet $branch + if ($LASTEXITCODE -ne 0) { + Write-Output "Creating new orphan branch" + } } + git switch --recurse-submodules ($LASTEXITCODE -ne 0 ? '--orphan' : $null) $branch - if ($(Test-Path -Path $VersionPath.FullName)) { + if (Test-Path $VersionPath) { Write-Output "Removing existing docs in $($VersionPath.FullName)" Remove-Item -Recurse -Path $VersionPath.FullName } - if ($(Test-Path -Path ".nojekyll") -eq $False) { + if (!(Test-Path ".nojekyll")) { Write-Output "Creating a .nojekyll file" Write-Output "" > .nojekyll git add .nojekyll } Write-Output "Moving $($VersionPath.FullName)-new back to original location" Move-Item "$($VersionPath.FullName)-new" $VersionPath.FullName -} -finally { - Write-Output "Leaving '$RepoPath'" + +} finally { + Write-Output "Leaving $RepoName" Pop-Location } diff --git a/steps/get-next-package-version.ps1 b/steps/get-next-package-version.ps1 index 4d47290f..4b684570 100644 --- a/steps/get-next-package-version.ps1 +++ b/steps/get-next-package-version.ps1 @@ -1,38 +1,26 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, [string]$VariableName = "GitVersion", - [string]$GitVersionConfigPath = $Null + [string]$GitVersionConfigPath ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - +Write-Output "Entering $RepoName" +Push-Location $RepoName try { - Write-Output "Installing gitversion" dotnet tool install --global GitVersion.Tool --version 5.* - if ($Null -ne $GitVersionConfigPath) { - $GitVersionOutput = dotnet-gitversion /config $GitVersionConfigPath - } - else { - $GitVersionOutput = dotnet-gitversion - } - - Write-Output $GitVersionOutput - - Write-Output "Setting gitversion as '$VariableName'" - Set-Variable -Name $VariableName -Value $(Write-Output $GitVersionOutput | ConvertFrom-Json) -Scope 1 + $GitVersionOutput = dotnet-gitversion ($GitVersionConfigPath ? '/config', $GitVersionConfigPath : $null) -} -finally { + Write-Output "Setting $VariableName to:" + Write-Output $GitVersionOutput + Set-Variable -Scope 1 -Name $VariableName -Value $($GitVersionOutput | ConvertFrom-Json) - Write-Output "Leaving '$RepoPath'" +} finally { + Write-Output "Leaving $RepoName" Pop-Location - } diff --git a/steps/get-pull-requests.ps1 b/steps/get-pull-requests.ps1 index 7e7dd9c3..373167ad 100644 --- a/steps/get-pull-requests.ps1 +++ b/steps/get-pull-requests.ps1 @@ -3,15 +3,11 @@ param ( [string]$RepoName, [Parameter(Mandatory=$true)] [string]$OrgName, - [string]$VariableName = "PullRequestIds", - [string]$GitHubToken + [string]$Branch = "main", + [string]$SetVariable = "PullRequestIds" ) - -# This token is used by the gh command. -Write-Output "Setting GITHUB_TOKEN" -$env:GITHUB_TOKEN="$GitHubToken" - -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true $Collaborators = gh api /repos/$OrgName/$RepoName/collaborators | ConvertFrom-Json @@ -30,12 +26,13 @@ function Test-Approval { [Object[]]$Reviews, [Object[]]$RequestedReviewers ) + $InformationPreference = 'Continue' # First, check if all required reviewers approved the PR; fail early if not foreach ($reviewer in $RequestedReviewers) { if ($Reviews | Where-Object { $_.user.id -eq $reviewer.id -and $_.state -ne 'APPROVED' }) { Write-Information "The pull request is not approved by the requested reviewer: $($reviewer.login)" - return $False + return $false } } @@ -43,11 +40,11 @@ function Test-Approval { $approvals = ($Reviews | Where-Object { $_.state -eq 'APPROVED' -and (Test-WriteAccess $_.user.id) }).user.login if ($approvals) { Write-Information "The creator doesn't have write access, but the pull request has been approved by: $approvals" - return $True + return $true } Write-Information "The creator doesn't have write access, and the pull request is not approved by anyone with write access to the repository" - return $False + return $false } function Test-Pr { @@ -69,43 +66,28 @@ function Test-Pr { } } -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { +$Ids = gh pr list -R $OrgName/$RepoName -B $Branch --json number,isDraft --jq '.[]|select(.isDraft|not).number' +if ($Ids) { + $ValidIds = @() - $Ids = gh pr list -B main --json number,isDraft --jq '.[]|select(.isDraft|not).number' - if ($Null -ne $Ids) { - $ValidIds = @() - - foreach ($Id in $Ids) { - # Only select PRs which are eligeble for automation. - Write-Output "Checking PR #$Id" - if (Test-Pr $RepoName $Id) { - $ValidIds += $Id - } - } - - if ($ValidIds.Count -gt 0) { - Write-Output "Pull request ids are: $([string]::Join(",", $ValidIds))" - Set-Variable -Name $VariableName -Value $ValidIds -Scope Global - } - else { - Write-Output "No pull requests to be checked." - Set-Variable -Name $VariableName -Value @(0) -Scope Global + foreach ($Id in $Ids) { + # Only select PRs which are eligeble for automation. + Write-Output "Checking PR #$Id" + if (Test-Pr $RepoName $Id) { + $ValidIds += $Id } + } - } else { - + if ($ValidIds.Count -gt 0) { + Write-Output "Pull request ids are: $([string]::Join(",", $ValidIds))" + Set-Variable -Scope 1 -Name $SetVariable -Value $ValidIds + } + else { Write-Output "No pull requests to be checked." - Set-Variable -Name $VariableName -Value @(0) -Scope Global - + Set-Variable -Scope 1 -Name $SetVariable -Value @(0) } -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location +} else { + Write-Output "No pull requests to be checked." + Set-Variable -Scope 1 -Name $SetVariable -Value @(0) } - diff --git a/steps/has-changed.ps1 b/steps/has-changed.ps1 index 88813995..9aa2ae00 100644 --- a/steps/has-changed.ps1 +++ b/steps/has-changed.ps1 @@ -1,35 +1,12 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - Write-Output "Checking Changes" - $changes = $(git status -s) - - Write-Output "There are '$($changes.count)' changes:" - foreach ($change in $changes) { - Write-Output "`t$change" - } - - if ($changes.count -gt 0) { - exit 0 - } - else { - exit 1 - } - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location +$changes = $(git -C $RepoName status --porcelain) +Write-Output "There are $($changes.Count) changes:" +Write-Output $changes -} +exit $changes.count -gt 0 ? 0 : 1 diff --git a/steps/merge-pr.ps1 b/steps/merge-pr.ps1 index ca5df8cb..700d2d35 100644 --- a/steps/merge-pr.ps1 +++ b/steps/merge-pr.ps1 @@ -5,40 +5,22 @@ param ( [string]$OrgName, [Parameter(Mandatory=$true)] [int]$PullRequestId, - [bool]$DryRun = $False + [bool]$DryRun ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -. ./constants.ps1 -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - if ($PullRequestId -eq 0) { - - Write-Output "Not running for a PR" - exit 0 - - } - - $PrTitle = gh pr view $PullRequestId --json number,headRefName,baseRefName,title -t '#{{.number}} {{.headRefName}}->{{.baseRefName}} : {{.title}}' - - Write-Output "Merging PR $PrTitle" - $Command = {gh api /repos/$OrgName/$RepoName/pulls/$PullRequestId/merge -X PUT -f "commit_title=Merged Pull Request '$PrTitle'"} - if ($DryRun -eq $False) { - & $Command - } - else { - Write-Output "Dry run - not executing the following: $Command" - } - +if ($PullRequestId -eq 0) { + Write-Output "Not running for a PR" + exit } -finally { - Write-Output "Leaving '$RepoPath'" - Pop-Location +$PrTitle = gh pr view -R $OrgName/$RepoName $PullRequestId --json number,headRefName,baseRefName,title -t '#{{.number}} {{.headRefName}}->{{.baseRefName}}: {{.title}}' +if ($DryRun) { + Write-Output "Dry run - not merging" +} else { + Write-Output "Merging PR $PrTitle" + gh api /repos/$OrgName/$RepoName/pulls/$PullRequestId/merge -X PUT -f "commit_title=Merged Pull Request '$PrTitle'" } diff --git a/steps/package-update-required.ps1 b/steps/package-update-required.ps1 index 0dac58e1..11c6fff5 100644 --- a/steps/package-update-required.ps1 +++ b/steps/package-update-required.ps1 @@ -1,4 +1,3 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, @@ -6,36 +5,12 @@ param ( [string]$Version ) -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - Write-Output "Checking if update is required" - $Tags = $(git tag --list) - - foreach ($Tag in $Tags) { - - if ($Tag -eq $Version) { - - Write-Output "Version '$Version' already present, no update needed" - # This is not treated as an error, but an indication that the result - # is false. - exit 1 - - } - - } +Write-Output "Checking if update is required" +git -C $RepoName show-ref --quiet --tags $Version +if ($LASTEXITCODE -gt 0) { exit 0 - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location - } +Write-Output "Version '$Version' already present, no update needed" +exit 1 diff --git a/steps/pull-request-to-main.ps1 b/steps/pull-request-to-main.ps1 deleted file mode 100644 index c98d7398..00000000 --- a/steps/pull-request-to-main.ps1 +++ /dev/null @@ -1,45 +0,0 @@ - -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$Message, - [bool]$DryRun = $False -) - -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - $CurrentBranch = $(git rev-parse --abbrev-ref HEAD) - - Write-Output "Getting PRs from '$CurrentBranch' to 'main'" - $Prs = gh pr list -B main -H $CurrentBranch --json number --jq '.[].number' - - Write-Output "There are '$($Prs.Count)' PRs" - - if ($Prs.Count -eq 0) { - Write-Output "Creating pull request" - $Command = {gh pr create --title $Message --body $Message -H $CurrentBranch} - if ($DryRun -eq $False) { - & $Command - } - else { - Write-Output "Dry run - not executing the following: $Command" - } - } - else { - Write-Output "A PR already exists for this branch." - } - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location - -} - diff --git a/steps/pull-request.ps1 b/steps/pull-request.ps1 new file mode 100644 index 00000000..6c008854 --- /dev/null +++ b/steps/pull-request.ps1 @@ -0,0 +1,32 @@ +param ( + [Parameter(Mandatory=$true)] + [string]$RepoName, + [Parameter(Mandatory=$true)] + [string]$Message, + [Parameter(Mandatory=$true)] + [string]$From, + [string]$To = "main", + [bool]$DryRun +) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true + +Write-Output "Entering $RepoName" +Push-Location $RepoName +try { + Write-Output "Getting PRs from '$From' to '$To'" + $PRs = gh pr list -H $From -B $To --json number --jq '.[].number' + + Write-Output "There are '$($Prs.Count)' PRs" + + if ($PRs.Count -gt 0) { + Write-Output "A PR already exists for this branch." + } else { + gh pr create -H $From -B $To --title $Message --body $Message ($DryRun ? '--dry-run' : $null) + } + +} finally { + Write-Output "Leaving $RepoName" + Pop-Location +} + diff --git a/steps/push-changes.ps1 b/steps/push-changes.ps1 index c9fd2b78..9217d203 100644 --- a/steps/push-changes.ps1 +++ b/steps/push-changes.ps1 @@ -1,41 +1,14 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$Branch, - [bool]$DryRun = $False + [bool]$Force, + [bool]$DryRun ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - Write-Output "Pulling" - $Command = "git pull --rebase origin $Branch" - if ($DryRun -eq $False) { - Invoke-Expression $Command - } - else { - Write-Output "Dry run - not executing the following: $Command" - } - - Write-Output "Pushing" - $Command = "git push origin $Branch" - if ($DryRun -eq $False) { - Invoke-Expression $Command - } - else { - Write-Output "Dry run - not executing the following: $Command" - } - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location - +if ($DryRun) { + Write-Output "Dry run - not pushing" +} else { + git -C $RepoName push ($Force ? '--force-with-lease' : $null) origin HEAD } diff --git a/steps/run-repo-script.ps1 b/steps/run-repo-script.ps1 deleted file mode 100644 index e6d4932a..00000000 --- a/steps/run-repo-script.ps1 +++ /dev/null @@ -1,58 +0,0 @@ -param ( - [Parameter(Mandatory=$true)] - [string]$RepoName, - [Parameter(Mandatory=$true)] - [string]$OrgName, - [Parameter(Mandatory=$true)] - [string]$ScriptName, - $Options = @{}, - [bool]$DryRun = $False -) - -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -$Script = [IO.Path]::Combine($RepoPath, "ci", $ScriptName) - -# Get the named parameters from the script file. -$ScriptParameters = (Get-Command -Name $Script).Parameters - -$Parameters = @{} - -# Add all the parameters that are available in the options. -foreach ($Option in $Options.GetEnumerator()) { - if ($ScriptParameters.ContainsKey($Option.Key) -and $Option.Value -ne "") { - Write-Output "Adding parameter '$($Option.Key)'" - $Parameters.Add($Option.Key, $Option.Value) - } -} - -# If there are keys required, add these too. -if ($Null -ne $Options.Keys) { - foreach ($Key in $Options.Keys.GetEnumerator()) { - if ($ScriptParameters.ContainsKey($Key.Key) -and $Key.Value -ne "") { - Write-Output "Adding parameter '$($Key.Key)'" - $Parameters.Add($Key.Key, $Key.Value) - } - } -} -# If the repo name is required, then add that too. -if ($ScriptParameters.ContainsKey("RepoName")) { - Write-Output "Adding parameter RepoName" - $Parameters.Add("RepoName", $RepoName) -} -# If the org name is required, then add that too. -if ($ScriptParameters.ContainsKey("OrgName")) { - Write-Output "Adding parameter OrgName" - $Parameters.Add("OrgName", $OrgName) -} -# If the script accepts a dryrun option, then add it. -if ($ScriptParameters.ContainsKey("DryRun")) { - Write-Output "Adding parameter DryRun" - $Parameters.Add("DryRun", $DryRun) -} - -Write-Output "Running script '$Script'." - -. $Script @Parameters - -exit $LASTEXITCODE diff --git a/steps/run-script.ps1 b/steps/run-script.ps1 new file mode 100644 index 00000000..8113258d --- /dev/null +++ b/steps/run-script.ps1 @@ -0,0 +1,23 @@ +param ( + [Parameter(Mandatory=$true,Position=0)] + [string]$Script, + [Parameter(Position=1)] + $Options = @{} +) +$ErrorActionPreference = "Stop" + +$cmd = Get-Command -Name $Script +if (!$cmd.Parameters) { + throw "Failed to load command parameters (check for syntax errors): $Script" +} + +$Parameters = @{} + +foreach ($opt in $Options.GetEnumerator()) { + if ($cmd.Parameters.ContainsKey($opt.Key) -and $opt.Value) { + $Parameters[$opt.Key] = $opt.Value + } +} + +Write-Output "Running '$Script' with parameters: $($Parameters.psbase.Keys)" +. $Script @Parameters diff --git a/steps/update-sub-modules.ps1 b/steps/update-sub-modules.ps1 index 06388365..49781e88 100644 --- a/steps/update-sub-modules.ps1 +++ b/steps/update-sub-modules.ps1 @@ -1,29 +1,9 @@ - param ( [Parameter(Mandatory=$true)] [string]$RepoName ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) - -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath - -try { - - Write-Output "Merging any changes from main" - git merge origin/main - - Write-Output "Cloning Submodules" - git submodule update --init --recursive - - Write-Output "Updating Submodules" - git submodule foreach 'git checkout main && git pull origin' - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location - -} +Write-Output "Updating Submodules" +git -C $RepoName submodule update --remote diff --git a/steps/update-tag.ps1 b/steps/update-tag.ps1 index d6fcf76c..e423e683 100644 --- a/steps/update-tag.ps1 +++ b/steps/update-tag.ps1 @@ -5,43 +5,21 @@ param ( [string]$OrgName, [Parameter(Mandatory=$true)] [string]$Tag, - [bool]$DryRun = $False + [bool]$DryRun ) -$RepoPath = [IO.Path]::Combine($pwd, $RepoName) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true +Write-Output "Tagging '$Tag'" +git -C $RepoName tag $Tag -Write-Output "Entering '$RepoPath'" -Push-Location $RepoPath +if ($DryRun) { + Write-Output "Dry run - not pushing tag '$Tag'" + Write-Output "Dry run - not creating a release" +} else { + Write-Output "Pushing tag '$Tag'" + git -C $RepoName push origin $Tag -try { - - Write-Output "Tagging with '$Tag'" - git tag $Tag - - Write-Output "Pushing tag" - $Command = "git push origin $Tag" - if ($DryRun -eq $False) { - Invoke-Expression $Command - } - else { - Write-Output "Dry run - not executing the following: $Command" - } - - # When creating the release, auto-generate the release notes from the - # PRs that are included in the changes. Write-Output "Creating a GitHub release" - $Command = {gh api /repos/$OrgName/$RepoName/releases -X POST -f "tag_name=$Tag" -F "generate_release_notes=true" -f "name=Version $Tag"} - if ($DryRun -eq $False) { - & $Command - } - else { - Write-Output "Dry run - not executing the following: $Command" - } - -} -finally { - - Write-Output "Leaving '$RepoPath'" - Pop-Location - + gh api /repos/$OrgName/$RepoName/releases -X POST -f "tag_name=$Tag" -F "generate_release_notes=true" -f "name=Version $Tag" } diff --git a/steps/upload-release-assets.ps1 b/steps/upload-release-assets.ps1 index a390322d..0e670779 100644 --- a/steps/upload-release-assets.ps1 +++ b/steps/upload-release-assets.ps1 @@ -5,44 +5,30 @@ param ( [string]$OrgName, [Parameter(Mandatory=$true)] [string]$Tag, - [bool]$DryRun = $False + [bool]$DryRun ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true -$PackagePath = [IO.Path]::Combine($pwd, "package") +$files = Get-ChildItem "package" -Write-Output "Entering '$PackagePath'" -Push-Location $PackagePath - -try { - - $files = Get-ChildItem "." +if ($files.count -eq 0 -or ($files.Count -eq 1 -and $files[0].Name -eq "dummy.txt")) { + Write-Output "No files to upload." + return +} - if ($files.count -eq 0 -or ($files.Count -eq 1 -and $files[0].Name -eq "dummy.txt")) { - Write-Output "No files to upload." - return +foreach ($file in $files) { + if ($file.PSIsContainer) { + $archive = "package/$($file.Name).zip" + Write-Output "Compressing '$($file.Name)' to '$archive'" + Compress-Archive -Path $file -DestinationPath $archive + $file = Get-Item $archive } - foreach ($file in $files) { - - if ($file.Attributes.Equals([System.IO.FileAttributes]::Directory)) { - Write-Output "Compressing $($file.Name)" - Compress-Archive -Path $file -DestinationPath "$($file).zip" - $file = Get-Item "$($file).zip" - } + if ($DryRun) { + Write-Output "Dry run - not uploading '$($file.Name)'" + } else { Write-Output "Uploading $($file.Name)" - $Command = {gh release upload $Tag $file.Name --repo https://github.com/$OrgName/$RepoName} - if ($DryRun -eq $False) { - & $Command - } - else { - Write-Output "Dry run - not executing the following: $Command" - } + gh release upload $Tag $file --repo https://github.com/$OrgName/$RepoName } - -} -finally { - - Write-Output "Leaving '$PackagePath'" - Pop-Location - } From 293ece4577e58eabd59c269581f552422b6310dd Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Mon, 19 Aug 2024 15:25:35 +0300 Subject: [PATCH 02/17] readme: update documentation --- README.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 85b3d575..7cd86c39 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The following rules are common across the organisation. - GitHub triggers initiate the stages. - Use of GitHub actions and other platform specific features are minimised to enable portability of CI/CD. For example, cloning a repository is performed in PowerShell via a generic command line that works on Linux, Windows, and Mac rather than in a GitHub action. This ensures the PowerShell script can be tested outside a CI/CD deployment environment. - C/C++ is compiled using CMake on all platforms. -- Versioning is performed using the most appropriate method for the target language. For example, GitVersion used for .NET +- Versioning is handled by GitVersion. - Common PowerShell scripts are contained in this repository. Repository specific scripts are contained under the root `ci` folder. - Code that is in the `main` branch has passed all relevant tests. This is a critical gate to avoid repeating tests. - The package output from a repository in the organisation is used as the input to other packages in the same organisation via the relevant package manager. This ensures that the organisations published packages are treated just like any other dependency and are not given special treatment. @@ -256,23 +256,23 @@ Each of the triggers shown is described at a high level. More detail is provided ### Nightly Data File Change -When data files used by the packages change new properties might be added or current ones deprecated. The strongly type accessors for the language might therefore need to change resulting in a new version of the resulting package. Every night, data files are fetched, and the auto generated strongly typed accessor code is updated. Any changes are then committed to a branch and a pull request to `main` is created. +When data files used by the packages change new properties might be added or current ones deprecated. The strongly type accessors for the language might therefore need to change resulting in a new version of the resulting package. Every night, data files are fetched, and the auto generated strongly typed accessor code is updated. Any changes are then committed to a branch and a pull request to the target branch is created. ### Nightly Package Dependency Update and Nightly Sub-Module Update -Every night any dependencies of the package are updated automatically to the latest patch version of that package or `main` branch commit of the sub-module. This includes any dependencies on packages within the organisation. Where changes are identified a branch is created and a PR to `main` is initiated. The tests associated with any PR to `main` will identify any failures for engineers to address before the updated versions can be used. +Every night any dependencies of the package are updated automatically to the latest patch version of that package or the latest commit of the sub-module's specified branch. This includes any dependencies on packages within the organisation. Where changes are identified a branch is created and a PR to the target branch is initiated. The tests associated with any PR to the target branch will identify any failures for engineers to address before the updated versions can be used. -### PR to Main +### Pull Requests All tests associated with the repository are run only at this point to avoid repetition. Code can only be present in the main branch of the repository if all tests have passed. -PRs to `main` can only be initiated by a project Contributor, Administrator, or GitHub actions. +Pull Requests can only be initiated by a project Contributor, Administrator, or GitHub actions. -### Nightly Main Package Publish +### Nightly Package **This job should only be run once all the other nightly jobs have completed.** -Any changes to the `main` branch are published automatically on a nightly basis as a new package at the target package manager environment. +Any changes to the target branch are published automatically on a nightly basis as a new package at the target package manager environment. This can also be run manually against any branch. If the branch is not the main branch, then the generated version for the package will be a prerelease version, following semantic versioning. @@ -280,6 +280,16 @@ This can also be run manually against any branch. If the branch is not the main Any changes to the `main` branch are used to generate the latest documentation. This is then published to the `gh-pages` branch of the reposiory. +### Nightly Pipeline + +This is a wrapper workflow that's not present in `common-ci`. It calls other nightly workflows (some in paralell, others sequentially) as part of a single workflow to preserve the trigger metadata between them, and allow running the entire Nightly workflows suite from any branch. This approach is required, because unlike PowerShell scripts, that can be parametrized to run from any branch, some of the actions that the workflows call (e.g. [publish-unit-test-result-action](https://github.com/EnricoMi/publish-unit-test-result-action)) use the branch from GitHub's metadata. + +The wrapper should usually just be a combination of other nightly workflows, with dependencies between stages specified using the `needs` job property instead of `workflow_run` triggers used previously. + +One limitation of such approach is that GitHub's scheduled pipelines always run from `main`, but we need to be able to schedule them on any branch. To work around this limitation a separate scheduler workflow is used. Its only task is to trigger the Nightly Pipeline workflow from the specified branch, using the GitHub API. + +The unified nightly pipeline is opt-in. Only repositories that need to run the nightly suite from branches other than `main` have to use it. Other repositories can continue using the previous system, based on `workflow_run` triggers. + ### Monthly Copyright Update If the copyright is updated, the source code files are all updated with the latest copyright in the header of the file. @@ -294,7 +304,7 @@ The Data File Change trigger creates new auto generated code for the strongly ty #### Update dependency -A package that the organisation uses is updated to a new version within the same `major.minor` version. The nightly dependency check will pick up on the new version and create a branch and associated PR to main. The changes will then propagate to the published packages if the tests executed for any PR to main pass. +A package that the organisation uses is updated to a new version within the same `major.minor` version. The nightly dependency check will pick up on the new version and create a branch and associated PR to main. The changes will then propagate to the published packages if the tests executed for any PR pass. #### Organisation package update From 70297cf0883769c96735f4fb380c56e138e53782 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Mon, 7 Oct 2024 16:19:25 +0300 Subject: [PATCH 03/17] cxx: add gitversion config --- cxx/get-next-package-version.ps1 | 4 ++-- cxx/gitversion.yml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 cxx/gitversion.yml diff --git a/cxx/get-next-package-version.ps1 b/cxx/get-next-package-version.ps1 index cd3083d4..3d9d0c88 100644 --- a/cxx/get-next-package-version.ps1 +++ b/cxx/get-next-package-version.ps1 @@ -6,6 +6,6 @@ param ( ) $ErrorActionPreference = "Stop" -./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName GitVersion +./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName GitVersion -GitVersionConfigPath cxx/gitversion.yml -Set-Variable -Scope 1 -Name $VariableName -Value $GitVersion.MajorMinorPatch +Set-Variable -Scope 1 -Name $VariableName -Value $GitVersion.SemVer diff --git a/cxx/gitversion.yml b/cxx/gitversion.yml new file mode 100644 index 00000000..a3a2fc3c --- /dev/null +++ b/cxx/gitversion.yml @@ -0,0 +1,4 @@ +branches: + alpha: + regex: ^version/ + label: alpha From ad3092c45ca9dfac3392ba5207212584479beff2 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Mon, 7 Oct 2024 16:22:09 +0300 Subject: [PATCH 04/17] update-tag: mark alpha tags as pre-release --- nightly-publish.configure.ps1 | 2 +- steps/get-next-package-version.ps1 | 2 +- steps/update-tag.ps1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nightly-publish.configure.ps1 b/nightly-publish.configure.ps1 index e9b44a4a..cca19789 100644 --- a/nightly-publish.configure.ps1 +++ b/nightly-publish.configure.ps1 @@ -26,7 +26,7 @@ Write-Output "::endgroup::" Write-Output "::group::Get Next Package Version" ./steps/run-script.ps1 ./$RepoName/ci/get-next-package-version.ps1 @{RepoName = $RepoName; VariableName = "Version"} Write-Output version=$Version -if (!($Version -match '^(\d+)\.(\d+)\.(\d+)(\.\d+)?(-SNAPSHOT)?$')) { +if (!($Version -match '^(\d+)\.(\d+)\.(\d+)(\.\d+)?(-SNAPSHOT|-alpha\.\d+)?$')) { Write-Error "Version '$Version' isn't valid" exit 1 } diff --git a/steps/get-next-package-version.ps1 b/steps/get-next-package-version.ps1 index 4b684570..9a958a50 100644 --- a/steps/get-next-package-version.ps1 +++ b/steps/get-next-package-version.ps1 @@ -11,7 +11,7 @@ Write-Output "Entering $RepoName" Push-Location $RepoName try { Write-Output "Installing gitversion" - dotnet tool install --global GitVersion.Tool --version 5.* + dotnet tool install --global GitVersion.Tool --version 6.* $GitVersionOutput = dotnet-gitversion ($GitVersionConfigPath ? '/config', $GitVersionConfigPath : $null) diff --git a/steps/update-tag.ps1 b/steps/update-tag.ps1 index e423e683..b1d48a62 100644 --- a/steps/update-tag.ps1 +++ b/steps/update-tag.ps1 @@ -21,5 +21,5 @@ if ($DryRun) { git -C $RepoName push origin $Tag Write-Output "Creating a GitHub release" - gh api /repos/$OrgName/$RepoName/releases -X POST -f "tag_name=$Tag" -F "generate_release_notes=true" -f "name=Version $Tag" + gh -R $OrgName/$RepoName release create --verify-tag --generate-notes ($Tag -cmatch '-SNAPSHOT$|-alpha(\.\d+)?$' ? '--prerelease' : $null) $Tag } From 7cf60e83fe80f37769a45a8391392c285c076f89 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Mon, 7 Oct 2024 17:11:48 +0300 Subject: [PATCH 05/17] cxx/dotnet: fix gitversion config path --- cxx/get-next-package-version.ps1 | 2 +- dotnet/get-next-package-version.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cxx/get-next-package-version.ps1 b/cxx/get-next-package-version.ps1 index 3d9d0c88..79525d5b 100644 --- a/cxx/get-next-package-version.ps1 +++ b/cxx/get-next-package-version.ps1 @@ -6,6 +6,6 @@ param ( ) $ErrorActionPreference = "Stop" -./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName GitVersion -GitVersionConfigPath cxx/gitversion.yml +./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName GitVersion -GitVersionConfigPath $PWD/cxx/gitversion.yml Set-Variable -Scope 1 -Name $VariableName -Value $GitVersion.SemVer diff --git a/dotnet/get-next-package-version.ps1 b/dotnet/get-next-package-version.ps1 index 0014e5cb..469b2272 100644 --- a/dotnet/get-next-package-version.ps1 +++ b/dotnet/get-next-package-version.ps1 @@ -5,7 +5,7 @@ param ( [string]$VariableName = "Version" ) -./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName $VariableName -GitVersionConfigPath dotnet/gitversion.yml +./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName $VariableName -GitVersionConfigPath $PWD/dotnet/gitversion.yml $SemVer = (Get-Variable -Name $VariableName).Value.SemVer Set-Variable -Name $VariableName -Value $SemVer -Scope Global From 52c4d9fd0c584556928fd3928df14b0ff033d3ee Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Wed, 9 Oct 2024 11:47:34 +0300 Subject: [PATCH 06/17] package-update/data-file-change: small fixes --- .github/workflows/nightly-package-update.yml | 1 - nightly-data-file-change.ps1 | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/nightly-package-update.yml b/.github/workflows/nightly-package-update.yml index 0cb39c2b..b3eb2345 100644 --- a/.github/workflows/nightly-package-update.yml +++ b/.github/workflows/nightly-package-update.yml @@ -55,7 +55,6 @@ jobs: -RepoName ${{ inputs.repo-name }} ` -OrgName ${{ inputs.org-name }} ` -Branch ${{ github.ref_name }} ` - -RunId ${{ github.run_id }} ` -GitHubUser ${{ inputs.user }} ` -GitHubEmail ${{ inputs.email }} ` -DryRun $DryRun diff --git a/nightly-data-file-change.ps1 b/nightly-data-file-change.ps1 index b2d61df2..3a76d9c0 100644 --- a/nightly-data-file-change.ps1 +++ b/nightly-data-file-change.ps1 @@ -33,7 +33,7 @@ Write-Output "::group::Fetch Assets" Write-Output "::endgroup::" Write-Output "::group::Generate Accessors" -./steps/run-script.ps1 ./tools/ci/generate-accessors.ps1 @{RepoName = tools, TargetRepo = $RepoName} +./steps/run-script.ps1 ./tools/ci/generate-accessors.ps1 @{RepoName = 'tools'; TargetRepo = $RepoName} Write-Output "::endgroup::" Write-Output "::group::Has Changed" From 828a60489aacbb80e78c9ffaa69169347d8e6211 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Wed, 9 Oct 2024 13:03:14 +0300 Subject: [PATCH 07/17] dotnet: write-error on build/test failure Exit code that is returned currently wasn't being caught by the calling scripts, as they expect PowerShell scripts to handle native exit codes themselves. So we do just that - throw an error if any of the tests failed. The caller's `$ErrorActionPreference` would then be able to catch it. Ideally `build-project-*.ps1` scripts should use `$PSNativeCommandUseErrorActionPreference` instead, but that'd be best done as part of overall refactor to make all scrips that call native commands use it. --- dotnet/build-project-core.ps1 | 4 +--- dotnet/build-project-framework.ps1 | 4 +--- dotnet/run-unit-tests.ps1 | 17 ++++++----------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/dotnet/build-project-core.ps1 b/dotnet/build-project-core.ps1 index a17b2e8a..fe19b672 100644 --- a/dotnet/build-project-core.ps1 +++ b/dotnet/build-project-core.ps1 @@ -16,7 +16,7 @@ Push-Location $RepoPath try { Write-Output "Building '$Name'" - dotnet build $ProjectDir -c $Configuration /p:Platform=$Arch /p:BuiltOnCI=true + dotnet build $ProjectDir -c $Configuration /p:Platform=$Arch /p:BuiltOnCI=true || Write-Error "build failed" } finally { @@ -24,5 +24,3 @@ finally { Pop-Location } - -exit $LASTEXITCODE diff --git a/dotnet/build-project-framework.ps1 b/dotnet/build-project-framework.ps1 index 6d735892..74759b4a 100644 --- a/dotnet/build-project-framework.ps1 +++ b/dotnet/build-project-framework.ps1 @@ -15,7 +15,7 @@ Push-Location $RepoPath try { - msbuild -t:build -restore $ProjectDir /p:Platform=$Arch /p:Configuration=$Configuration /p:BuiltOnCI=true -verbosity:minimal + msbuild -t:build -restore $ProjectDir /p:Platform=$Arch /p:Configuration=$Configuration /p:BuiltOnCI=true -verbosity:minimal || Write-Error "build failed" } finally { @@ -24,5 +24,3 @@ finally { Pop-Location } - -exit $LASTEXITCODE diff --git a/dotnet/run-unit-tests.ps1 b/dotnet/run-unit-tests.ps1 index 0f05cbe8..c773a835 100644 --- a/dotnet/run-unit-tests.ps1 +++ b/dotnet/run-unit-tests.ps1 @@ -1,4 +1,3 @@ - param( [Parameter(Mandatory=$true)] [string]$RepoName, @@ -19,9 +18,8 @@ $TestResultPath = [IO.Path]::Combine($RepoPath, "test-results", $OutputFolder, $ Write-Output "Entering '$RepoPath'" Push-Location $RepoPath -$ok = $true - try { + $ok = $true $skipPattern = "*performance*" Write-Output "Testing '$Name'" @@ -33,8 +31,7 @@ try { dotnet test $_.FullName --results-directory $TestResultPath --blame-crash -l "trx" || $($script:ok = $false) } } - } - else{ + } else { Write-Output "[$BuildMethod] ~> Looking for '$Filter' in directories like '$DirNameFormatForNotDotnet'" Get-ChildItem -Path $RepoPath -Recurse -File | ForEach-Object { if (($_.DirectoryName -like $DirNameFormatForNotDotnet -and $_.Name -notlike $skipPattern) -and ($_.Name -match "$Filter")) { @@ -44,12 +41,10 @@ try { } } -} -finally { - + if (!$ok) { + Write-Error "Tests failed" + } +} finally { Write-Output "Leaving '$RepoPath'" Pop-Location - } - -exit $ok ? 0 : 1 From 7dd09484cab0b5f22209d2bd1003d09b2b78c0db Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Thu, 10 Oct 2024 11:12:14 +0300 Subject: [PATCH 08/17] publish: allow publishing `version/*` branches --- nightly-publish.package.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nightly-publish.package.ps1 b/nightly-publish.package.ps1 index cdffaade..53cf187c 100644 --- a/nightly-publish.package.ps1 +++ b/nightly-publish.package.ps1 @@ -33,7 +33,7 @@ Write-Output "::group::Install Package From Artifact" Write-Output "::endgroup::" Write-Output "::group::Publish Packages" -if ($Branch -eq "main") { +if ($Branch -ceq "main" -or $Branch -clike "version/*") { ./steps/run-script.ps1 ./$RepoName/ci/publish-package.ps1 $Options } else { Write-Output "Not on the main branch, skipping publishing" From 49797e5bda5f98ee3dd4f079c6fdbc2a8299c948 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Fri, 11 Oct 2024 13:10:50 +0300 Subject: [PATCH 09/17] java: update gitversion config Switch to unified `x.y.z-alpha.n` versioning scheme instead of the `x.y.z-SNAPSHOT`. --- java/get-next-package-version.ps1 | 16 +++++----------- java/gitversion.yml | 10 +++------- java/publish-package-maven.ps1 | 2 +- java/readme.md | 4 ---- nightly-publish.configure.ps1 | 2 +- steps/update-tag.ps1 | 2 +- 6 files changed, 11 insertions(+), 25 deletions(-) diff --git a/java/get-next-package-version.ps1 b/java/get-next-package-version.ps1 index eda9ed53..bb89e567 100644 --- a/java/get-next-package-version.ps1 +++ b/java/get-next-package-version.ps1 @@ -1,17 +1,11 @@ param ( [Parameter(Mandatory=$true)] - [string]$VariableName, [string]$RepoName, - [string]$ProjectDir = "." + [Parameter(Mandatory=$true)] + [string]$VariableName ) +$ErrorActionPreference = "Stop" -$GitVersionConfigPath = [IO.Path]::Combine($pwd, "java", "gitversion.yml") - -./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName $VariableName -GitVersionConfigPath $GitVersionConfigPath - -$assemblySemVer = (Get-Variable -Name $VariableName).Value.AssemblySemVer -Set-Variable -Name $VariableName -Value $assemblySemVer -Scope Global - -exit $LASTEXITCODE - +./steps/get-next-package-version.ps1 -RepoName $RepoName -VariableName GitVersion -GitVersionConfigPath $PWD/java/gitversion.yml +Set-Variable -Scope 1 -Name $VariableName -Value $GitVersion.SemVer diff --git a/java/gitversion.yml b/java/gitversion.yml index 383e144a..a3a2fc3c 100644 --- a/java/gitversion.yml +++ b/java/gitversion.yml @@ -1,8 +1,4 @@ -# This format gives an 'AssemblySemVer' of x.y.z for main, and x.y.z-SNAPSHOT for develop -# where the value of z in develop is always one increment ahead of main. -assembly-versioning-format: '{Major}.{Minor}.{Patch}{PreReleaseLabelWithDash}' branches: - develop: - tag: 'SNAPSHOT' - increment: Patch - regex: ^develop$ + alpha: + regex: ^version/ + label: alpha diff --git a/java/publish-package-maven.ps1 b/java/publish-package-maven.ps1 index 95adee50..ff2b0a17 100644 --- a/java/publish-package-maven.ps1 +++ b/java/publish-package-maven.ps1 @@ -45,7 +45,7 @@ try { "-Dhttps.protocols=TLSv1.2" ` "-DfailIfNoTests=false" - if ($($Version.EndsWith("SNAPSHOT")) -eq $False) { + if ($Version -cnotmatch '-alpha(\.\d+)?$') { Write-Output "Releasing from Nexus to Maven central" mvn -B nexus-staging:release ` diff --git a/java/readme.md b/java/readme.md index 46fd8d3a..33a27162 100644 --- a/java/readme.md +++ b/java/readme.md @@ -35,7 +35,3 @@ The packages are then copied to and from two locations on the local machine: the ``` To build packages locally, we use the mvn deploy command with the -DskipRemoteStaging=true option, which ensures that packages are not deployed to a remote repository. Once the packages are built, the `publish-package-maven.ps1` script uses the `mvn nexus-staging:deploy-staged` command to stage the packages to the remote Nexus staging repository. - -## SNAPSHOTS -The snapshot repository is configured in the parent pom.xml file. We use gitversion.yml to append the "-SNAPSHOT" string to the version number when the pipeline runs from a develop branch. -The deployment process for snapshot versions is the same as that for release versions. We follow the same steps for testing and deploying both types of packages. \ No newline at end of file diff --git a/nightly-publish.configure.ps1 b/nightly-publish.configure.ps1 index cca19789..f216f382 100644 --- a/nightly-publish.configure.ps1 +++ b/nightly-publish.configure.ps1 @@ -26,7 +26,7 @@ Write-Output "::endgroup::" Write-Output "::group::Get Next Package Version" ./steps/run-script.ps1 ./$RepoName/ci/get-next-package-version.ps1 @{RepoName = $RepoName; VariableName = "Version"} Write-Output version=$Version -if (!($Version -match '^(\d+)\.(\d+)\.(\d+)(\.\d+)?(-SNAPSHOT|-alpha\.\d+)?$')) { +if (!($Version -cmatch '^(\d+)\.(\d+)\.(\d+)(\.\d+)?(-alpha(\.\d+)?)?$')) { Write-Error "Version '$Version' isn't valid" exit 1 } diff --git a/steps/update-tag.ps1 b/steps/update-tag.ps1 index b1d48a62..898a8180 100644 --- a/steps/update-tag.ps1 +++ b/steps/update-tag.ps1 @@ -21,5 +21,5 @@ if ($DryRun) { git -C $RepoName push origin $Tag Write-Output "Creating a GitHub release" - gh -R $OrgName/$RepoName release create --verify-tag --generate-notes ($Tag -cmatch '-SNAPSHOT$|-alpha(\.\d+)?$' ? '--prerelease' : $null) $Tag + gh -R $OrgName/$RepoName release create --verify-tag --generate-notes ($Tag -cmatch '-alpha(\.\d+)?$' ? '--prerelease' : $null) $Tag } From 9630142132bdbe9a3368cc943b24fe16eaf8d2f7 Mon Sep 17 00:00:00 2001 From: Eugene Dorfman Date: Sat, 12 Oct 2024 23:44:59 +0200 Subject: [PATCH 10/17] dotnet: add unit tests status logging --- dotnet/run-unit-tests.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotnet/run-unit-tests.ps1 b/dotnet/run-unit-tests.ps1 index c773a835..5f8351f5 100644 --- a/dotnet/run-unit-tests.ps1 +++ b/dotnet/run-unit-tests.ps1 @@ -29,6 +29,7 @@ try { if (($_.DirectoryName -like $DirNameFormatForDotnet -and $_.Name -notlike $skipPattern) -and ($_.Name -match "$Filter")) { Write-Output "Testing Assembly: '$_'" dotnet test $_.FullName --results-directory $TestResultPath --blame-crash -l "trx" || $($script:ok = $false) + Write-Output "dotnet test LastExitCode=$LASTEXITCODE" } } } else { @@ -37,6 +38,7 @@ try { if (($_.DirectoryName -like $DirNameFormatForNotDotnet -and $_.Name -notlike $skipPattern) -and ($_.Name -match "$Filter")) { Write-Output "Testing Assembly: '$_'" & vstest.console.exe $_.FullName /Logger:trx /ResultsDirectory:$TestResultPath || $($script:ok = $false) + Write-Output "vstest.console LastExitCode=$LASTEXITCODE" } } } From fa5be4cb3fd1f33176c0a5be6e233e7a869e51aa Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Mon, 4 Nov 2024 11:27:16 +0200 Subject: [PATCH 11/17] README: specify scheduler workflow name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7cd86c39..ffb752a1 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,7 @@ This is a wrapper workflow that's not present in `common-ci`. It calls other nig The wrapper should usually just be a combination of other nightly workflows, with dependencies between stages specified using the `needs` job property instead of `workflow_run` triggers used previously. -One limitation of such approach is that GitHub's scheduled pipelines always run from `main`, but we need to be able to schedule them on any branch. To work around this limitation a separate scheduler workflow is used. Its only task is to trigger the Nightly Pipeline workflow from the specified branch, using the GitHub API. +One limitation of such approach is that GitHub's scheduled pipelines always run from `main`, but we need to be able to schedule them on any branch. To work around this limitation a separate scheduler workflow (`nightly-pipeline-prerelease.yml`) is used. Its only task is to trigger the Nightly Pipeline workflow from the specified branch, using the GitHub API. The unified nightly pipeline is opt-in. Only repositories that need to run the nightly suite from branches other than `main` have to use it. Other repositories can continue using the previous system, based on `workflow_run` triggers. From 630493d3cfb507d961b37a446cf727ad8f4586ec Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Thu, 7 Nov 2024 13:24:41 +0200 Subject: [PATCH 12/17] java: publish prerelease versions to maven --- java/publish-package-maven.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/publish-package-maven.ps1 b/java/publish-package-maven.ps1 index ff2b0a17..48854e86 100644 --- a/java/publish-package-maven.ps1 +++ b/java/publish-package-maven.ps1 @@ -45,10 +45,10 @@ try { "-Dhttps.protocols=TLSv1.2" ` "-DfailIfNoTests=false" - if ($Version -cnotmatch '-alpha(\.\d+)?$') { + #if ($Version -cnotmatch '-alpha(\.\d+)?$') { - Write-Output "Releasing from Nexus to Maven central" - mvn -B nexus-staging:release ` + Write-Output "Releasing from Nexus to Maven central" + mvn -B nexus-staging:release ` -s $SettingsPath ` -f pom.xml ` -DXmx2048m ` @@ -58,7 +58,7 @@ try { "-DfailIfNoTests=false" - } + #} } else{ From 83b7ed2be55a5fcbea30c099e3c7ad11ea7d1606 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Wed, 8 Jan 2025 15:25:23 +0200 Subject: [PATCH 13/17] java: set ErrorActionPreference in package update --- java/run-update-dependencies.ps1 | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/java/run-update-dependencies.ps1 b/java/run-update-dependencies.ps1 index 69c0a095..312b93cf 100644 --- a/java/run-update-dependencies.ps1 +++ b/java/run-update-dependencies.ps1 @@ -1,4 +1,3 @@ - param( [Parameter(Mandatory=$true)] [string]$RepoName, @@ -6,6 +5,8 @@ param( [string]$Name, [switch]$AllowSnapshots ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true $RepoPath = [IO.Path]::Combine($pwd, $RepoName) @@ -23,5 +24,3 @@ finally { Pop-Location } - -exit $LASTEXITCODE From d98517e11b307a57e6ddc90157e854f95190a89c Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Fri, 10 Jan 2025 11:26:40 +0200 Subject: [PATCH 14/17] java: stop on publish errors Including native command errors. --- java/publish-package-maven.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/publish-package-maven.ps1 b/java/publish-package-maven.ps1 index 48854e86..ab692658 100644 --- a/java/publish-package-maven.ps1 +++ b/java/publish-package-maven.ps1 @@ -6,6 +6,8 @@ param ( [Parameter(Mandatory=$true)] [string]$Version ) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true $RepoPath = [IO.Path]::Combine($pwd, $RepoName) From c4c35da02cd29301e976ce72989c55c8cfc10455 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Fri, 10 Jan 2025 11:55:43 +0200 Subject: [PATCH 15/17] java: allow reflection in java.util for publish --- java/publish-package-maven.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/publish-package-maven.ps1 b/java/publish-package-maven.ps1 index ab692658..3c73bd88 100644 --- a/java/publish-package-maven.ps1 +++ b/java/publish-package-maven.ps1 @@ -38,6 +38,8 @@ try { Write-Output "Deploying to Nexus staging" + $env:MAVEN_OPTS='--add-opens=java.base/java.util=ALL-UNNAMED' + mvn -B nexus-staging:deploy-staged ` -s $SettingsPath ` -f pom.xml ` @@ -59,7 +61,6 @@ try { "-Dhttps.protocols=TLSv1.2" ` "-DfailIfNoTests=false" - #} } else{ From d83dc6d412689cb5508cbc4ba3a86ec3621bb43c Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Tue, 14 Jan 2025 09:44:27 +0200 Subject: [PATCH 16/17] compat: add temp copies for old workflow names --- .github/workflows/nightly-prs-to-main.yml | 112 +++++++ .github/workflows/nightly-publish-main.yml | 327 +++++++++++++++++++++ 2 files changed, 439 insertions(+) create mode 100644 .github/workflows/nightly-prs-to-main.yml create mode 100644 .github/workflows/nightly-publish-main.yml diff --git a/.github/workflows/nightly-prs-to-main.yml b/.github/workflows/nightly-prs-to-main.yml new file mode 100644 index 00000000..d74e093a --- /dev/null +++ b/.github/workflows/nightly-prs-to-main.yml @@ -0,0 +1,112 @@ +# +# TODO: switch all repos to the new file name (`nightly-publish.yml`), and delete this file +# +name: Nightly Pull Requests + +on: + workflow_call: + inputs: + repo-name: + type: string + required: true + org-name: + type: string + default: 51Degrees + user: + type: string + default: Automation51D + email: + type: string + default: 51DCI@51degrees.com + dryrun: + type: string + default: false + cache-assets: + type: boolean + common-ci-ref: + type: string + default: '' + prs: # Comma-separated list of pull request IDs to build + type: string + secrets: + token: + required: true + AcceptCHBrowserKey: + AcceptCHHardwareKey: + AcceptCHNoneKey: + AcceptCHPlatformKey: + ApiKey: + CsvUrl: + DeviceDetection: + DeviceDetectionUrl: + JavaGpgKeyPassphrase: + JavaPGP: + MavenSettings: + NPMAuthToken: + NginxCert: + NginxKey: + NginxJwtToken: + PypiToken: + TestBrowserResourceKey: + TestHardwareResourceKey: + TestLicenseKey: + TestNoSetHeaderResourceKey: + TestPlatformResourceKey: + TestPypiToken: + TestResourceKey: + TestSuperResourceKey: + UsePublishTests: + PearlLicense: + +defaults: + run: + shell: pwsh + working-directory: ./common + +jobs: + # Get all the pull requests that are going to main + GetPullRequests: + name: Get Pull Requests + runs-on: ubuntu-latest + steps: + - name: Checkout Common + uses: actions/checkout@v4 + with: + repository: ${{ inputs.org-name }}/common-ci + path: common + ref: ${{ inputs.common-ci-ref }} + + # Get all the pull requests into main, and output the ids + - name: Get Pull Requests + id: get_pull_requests + env: + PULL_REQUEST_IDS: ${{ inputs.prs }} + run: | + ./nightly-pull-requests.get-prs.ps1 ` + -RepoName ${{ inputs.repo-name }} ` + -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` + -GitHubToken ${{ secrets.token }} ` + -GitHubOutput $Env:GITHUB_OUTPUT + outputs: + pull_request_ids: ${{ steps.get_pull_requests.outputs.pull_request_ids }} + + # Run the common workflow on each pull request + NightlyPullRequest: + needs: GetPullRequests + strategy: + fail-fast: false + matrix: + id: ${{ fromJSON(needs.GetPullRequests.outputs.pull_request_ids) }} + name: Nightly PR - ${{ matrix.id }} + uses: ./.github/workflows/nightly-pull-request.yml + with: + repo-name: ${{ inputs.repo-name }} + org-name: ${{ inputs.org-name }} + pull-request-id: ${{ matrix.id }} + user: ${{ inputs.user }} + email: ${{ inputs.email }} + dryrun: ${{ inputs.dryrun }} + cache-assets: ${{ inputs.cache-assets }} + common-ci-ref: ${{ inputs.common-ci-ref }} + secrets: inherit diff --git a/.github/workflows/nightly-publish-main.yml b/.github/workflows/nightly-publish-main.yml new file mode 100644 index 00000000..8bb16a2d --- /dev/null +++ b/.github/workflows/nightly-publish-main.yml @@ -0,0 +1,327 @@ +# +# TODO: switch all repos to the new file name (`nightly-publish.yml`), and delete this file +# +name: Nightly Publish + +on: + workflow_call: + inputs: + repo-name: + type: string + required: true + build-platform: + type: string + default: ubuntu-latest + org-name: + type: string + default: 51degrees + user: + type: string + default: Automation51D + email: + type: string + default: "51DCI@51Degrees.com" + dryrun: + type: boolean + default: false + cache-assets: + type: boolean + default: false + common-ci-ref: + type: string + default: '' + secrets: + # PAT that we use to authenticate on behalf of GitHub Actions. + token: + required: true + AcceptCHBrowserKey: + AcceptCHHardwareKey: + AcceptCHNoneKey: + AcceptCHPlatformKey: + ApiKey: + CodeSigningKeyVaultName: + CodeSigningKeyVaultUrl: + CodeSigningKeyVaultClientId: + CodeSigningKeyVaultTenantId: + CodeSigningKeyVaultClientSecret: + CodeSigningKeyVaultCertificateName: + CodeSigningKeyVaultCertificateData: + CsvUrl: + DeviceDetection: + DeviceDetectionUrl: + JavaGpgKeyPassphrase: + JavaPGP: + MavenSettings: + NPMAuthToken: + NginxCert: + NginxKey: + NginxJwtToken: + PypiToken: + TestBrowserResourceKey: + TestHardwareResourceKey: + TestLicenseKey: + TestNoSetHeaderResourceKey: + TestPlatformResourceKey: + TestPypiToken: + TestResourceKey: + TestSuperResourceKey: + UsePublishTests: + DockerPassword: + DockerContainer: + DockerRegistry: + DockerUser: + PearlLicense: + +defaults: + run: + shell: pwsh + working-directory: ./common + +jobs: + Configure: + runs-on: ubuntu-latest + outputs: + asset_cache_key: ${{ steps.hash_asset_cache_key.outputs.hash }} + options: ${{ steps.configure.outputs.options }} + version: ${{ steps.configure.outputs.version }} + update_required: ${{ steps.configure.outputs.update_required }} + steps: + - name: Checkout Common + uses: actions/checkout@v4 + with: + repository: ${{ inputs.org-name }}/common-ci + path: common + ref: ${{ inputs.common-ci-ref }} + + - name: Hash the Asset Cache Key + id: hash_asset_cache_key + if: inputs.cache-assets + env: + KEY: ${{ toJSON(secrets) }} + run: | + $stringAsStream = [System.IO.MemoryStream]::new() + $writer = [System.IO.StreamWriter]::new($stringAsStream) + $writer.write($env:KEY) + $writer.Flush() + $stringAsStream.Position = 0 + "hash=$(Get-Date -Format FileDate)-$((Get-FileHash -InputStream $stringAsStream).Hash)" | Out-File $env:GITHUB_OUTPUT -Append + + - name: Configure + id: configure + run: | + ./nightly-publish.configure.ps1 ` + -RepoName ${{ inputs.repo-name }} ` + -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` + -GitHubToken ${{ secrets.token }} ` + -GitHubOutput $Env:GITHUB_OUTPUT ` + -GitHubUser ${{ inputs.user }} ` + -GitHubEmail ${{ inputs.email }} ` + -BuildPlatform ${{ inputs.build-platform }} + + PreBuild: + needs: Configure + strategy: + matrix: + options: ${{ fromJSON(needs.configure.outputs.options ) }} + name: Pre Build - ${{ matrix.options.name }} + runs-on: ${{ matrix.options.image }} + steps: + - name: Checkout Common + uses: actions/checkout@v4 + if: ${{ matrix.options.packagerequirement }} + with: + repository: ${{ inputs.org-name }}/common-ci + path: common + ref: ${{ inputs.common-ci-ref }} + + - name: Install Runtime + if: matrix.options.Language != '' && matrix.options.LanguageVersion != '' + uses: ./common/.github/actions/install_runtime + with: + language: ${{ matrix.options.Language }} + version: ${{ matrix.options.LanguageVersion }} + + - name: PreBuild + if: ${{ matrix.options.packagerequirement }} + env: + SECRETS: ${{ toJSON(secrets) }} + run: | + $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) + $Options = $(ConvertFrom-Json -AsHashtable '${{ toJSON(matrix.options) }}') + $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) + ./nightly-publish.prebuild.ps1 ` + -RepoName ${{ inputs.repo-name }} ` + -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` + -GitHubToken ${{ secrets.token }} ` + -Options $Options ` + -GitHubUser ${{ inputs.user }} ` + -GitHubEmail ${{ inputs.email }} ` + -DryRun $DryRun + + - name: Upload Package Artifact + uses: actions/upload-artifact@v4 + if: ${{ success() && matrix.options.packagerequirement }} + with: + name: package_${{ matrix.options.name }} + path: ${{ github.workspace }}/common/package-files + if-no-files-found: ignore + include-hidden-files: true + + + Build: + runs-on: ${{ inputs.build-platform }} + needs: [ PreBuild, Configure ] + steps: + - name: Checkout Common + uses: actions/checkout@v4 + with: + repository: ${{ inputs.org-name }}/common-ci + path: common + ref: ${{ inputs.common-ci-ref }} + + - name: Download Package Artifacts + if: ${{ success() }} + uses: actions/download-artifact@v4 + with: + path: ${{ github.workspace }}/common/package-files + + - name: Build + env: + SECRETS: ${{ toJSON(secrets) }} + run: | + $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) + $Options = @{} + $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) + $Options.Add("Version", "${{ needs.configure.outputs.version }}") + ./nightly-publish.build.ps1 ` + -RepoName ${{ inputs.repo-name }} ` + -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` + -GitHubToken ${{ secrets.token }} ` + -Options $Options ` + -GitHubUser ${{ inputs.user }} ` + -GitHubEmail ${{ inputs.email }} ` + -DryRun $DryRun + + - name: Upload Package Artifact + uses: actions/upload-artifact@v4 + if: ${{ success() }} + with: + name: package + path: ${{ github.workspace }}/common/package + if-no-files-found: ignore + include-hidden-files: true + + + Test: + needs: [ Build, Configure ] + strategy: + matrix: + options: ${{ fromJSON(needs.configure.outputs.options ) }} + name: Test - ${{ matrix.options.name }} + runs-on: ${{ matrix.options.image }} + + steps: + - name: Checkout Common + if: ${{ matrix.options.packagerequirement }} + uses: actions/checkout@v4 + with: + repository: ${{ inputs.org-name }}/common-ci + path: common + ref: ${{ inputs.common-ci-ref }} + + - name: Download Package Artifact + if: ${{ success() && matrix.options.packagerequirement }} + uses: actions/download-artifact@v4 + with: + name: package + path: ${{ github.workspace }}/common/package + + - name: Install Runtime + if: matrix.options.Language != '' && matrix.options.LanguageVersion != '' + uses: ./common/.github/actions/install_runtime + with: + language: ${{ matrix.options.Language }} + version: ${{ matrix.options.LanguageVersion }} + + - name: Restore Asset Cache + if: inputs.cache-assets + uses: actions/cache/restore@v4 + with: + key: ${{ needs.configure.outputs.asset_cache_key }} + path: ${{ github.workspace }}/common/assets + + - name: Test + if: ${{ success() && matrix.options.packagerequirement }} + env: + SECRETS: ${{ toJSON(secrets) }} + run: | + $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) + $Options = $(ConvertFrom-Json -AsHashtable '${{ toJSON(matrix.options) }}') + $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) + ./nightly-publish.test.ps1 ` + -RepoName ${{ inputs.repo-name }} ` + -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` + -GitHubToken ${{ secrets.token }} ` + -Options $Options ` + -GitHubUser ${{ inputs.user }} ` + -GitHubEmail ${{ inputs.email }} ` + -DryRun $DryRun + + - name: Save Asset Cache + if: inputs.cache-assets && always() + uses: actions/cache/save@v4 + with: + key: ${{ needs.configure.outputs.asset_cache_key }} + path: ${{ github.workspace }}/common/assets + + - name: Publish Test Results + if: ${{ always() && matrix.options.packagerequirement }} + uses: ./common/.github/actions/publish_test_results + with: + repo-name: ${{ inputs.repo-name }} + name: ${{ matrix.options.name }} + skip-unit: true + skip-performance: true + + Package: + runs-on: ${{ inputs.build-platform }} + needs: [ Test, Configure ] + if: ${{ success() && needs.configure.outputs.update_required == 'true' }} + steps: + + - name: Checkout Common + uses: actions/checkout@v4 + with: + repository: ${{ inputs.org-name }}/common-ci + path: common + ref: ${{ inputs.common-ci-ref }} + + - name: Download Package Artifact + if: ${{ success() }} + uses: actions/download-artifact@v4 + with: + name: package + path: ${{ github.workspace }}/common/package + + - name: Package + if: ${{ success() }} + env: + SECRETS: ${{ toJSON(secrets) }} + run: | + $DryRun = [bool]::Parse( "${{ inputs.dryrun }}" ) + $Options = @{} + $Options.Add("Keys", $(ConvertFrom-Json -AsHashtable $env:SECRETS)) + $Options.Add("Version", "${{ needs.configure.outputs.version }}") + ./nightly-publish.package.ps1 ` + -RepoName ${{ inputs.repo-name }} ` + -OrgName ${{ inputs.org-name }} ` + -Branch ${{ github.ref_name }} ` + -GitHubToken ${{ secrets.token }} ` + -Options $Options ` + -GitHubUser ${{ inputs.user }} ` + -GitHubEmail ${{ inputs.email }} ` + -DryRun $DryRun From fc42c33accc37e5452cd7c20bb16c5b31510f3e6 Mon Sep 17 00:00:00 2001 From: Alexander Popel Date: Tue, 14 Jan 2025 16:12:20 +0200 Subject: [PATCH 17/17] go: add performance tests runner --- go/run-performance-tests.ps1 | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 go/run-performance-tests.ps1 diff --git a/go/run-performance-tests.ps1 b/go/run-performance-tests.ps1 new file mode 100644 index 00000000..267d09e3 --- /dev/null +++ b/go/run-performance-tests.ps1 @@ -0,0 +1,43 @@ +param ( + [Parameter(Mandatory)][string]$RepoName, + [Parameter(Mandatory)][string]$OrgName, + [Parameter(Mandatory)][string]$Name, + [Parameter(Mandatory)][string]$Example, + [string]$ExamplesRepo, + [string]$Branch = "main" +) +$ErrorActionPreference = "Stop" +$PSNativeCommandUseErrorActionPreference = $true + +$summaryDir = New-Item -ItemType directory -Path $RepoName/test-results/performance-summary -Force + +Push-Location $RepoName +try { + if ($ExamplesRepo) { + Write-Host "Cloning examples..." + ./steps/clone-repo.ps1 -OrgName $OrgName -RepoName $ExamplesRepo -Branch $Branch + } + + Write-Host "Running performance test..." + go run $Example + + switch -File performance_report.log -Regex { + 'Average ([^ ]+) ms per' { $MsPerDetection = [double]$matches.1 } + 'Average ([^ ]+) detections per second' { $DetectionsPerSecond = [double]$matches.1 } + } + + if (-not $MsPerDetection -or -not $DetectionsPerSecond) { + Get-Content performance_report.log | Write-Error + } + + @{ + HigherIsBetter = @{ + DetectionsPerSecond = $DetectionsPerSecond + } + LowerIsBetter = @{ + AvgMillisecsPerDetection = $MsPerDetection + } + } | ConvertTo-Json | Out-File $summaryDir/results_$Name.json +} finally { + Pop-Location +}