feat(cli): E2E bash testing (#715)

Co-authored-by: FuzzB0t <[email protected]>

.github/workflows/test-e2e.yml

@@ -0,0 +1,28 @@
+name: Test-e2e
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          cache: npm
+          node-version: '20.5.1'
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - run: |
+          npm ci
+          npm run build
+
+      - name: Initialize submodules
+        run: git submodule update --init --recursive
+
+      - name: E2E Tests
+        run: cd ./packages/cli && npm run test-e2e

.github/workflows/test-snx-ci.yml

@@ -0,0 +1,28 @@
+name: Test-snx-ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          cache: npm
+          node-version: '20.5.1'
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - run: |
+          npm ci
+          npm run build
+
+      - name: Initialize submodules
+        run: git submodule update --init --recursive
+
+      - name: Synthetix CI Downstream
+        run: cd ./packages/cli && npm run test-snx-ci

.gitmodules

@@ -9,3 +9,19 @@
 [submodule "packages/contracts/lib/openzeppelin-contracts"]
 	path = packages/contracts/lib/openzeppelin-contracts
 	url = https://github.com/openzeppelin/openzeppelin-contracts
+
+[submodule "packages/cli/test/e2e/helpers/bats"]
+	path = packages/cli/test/e2e/helpers/bats
+	url = https://github.com/bats-core/bats.git
+
+[submodule "packages/cli/test/e2e/helpers/bats-support"]
+	path = packages/cli/test/e2e/helpers/bats-support
+	url = https://github.com/bats-core/bats-support.git
+
+[submodule "packages/cli/test/e2e/helpers/bats-assert"]
+	path = packages/cli/test/e2e/helpers/bats-assert
+	url = https://github.com/bats-core/bats-assert.git
+
+[submodule "packages/cli/test/e2e/helpers/bats-file"]
+	path = packages/cli/test/e2e/helpers/bats-file
+	url = https://github.com/bats-core/bats-file.git

examples/sample-foundry-project/cannonfile.toml

@@ -20,6 +20,5 @@ value = "2929372"
 [contract.greeter]
 artifact = "Greeter"
 args = ["<%= settings.msg %>"]
-value = "<%= settings.value %>"
 libraries.Library = "<%= contracts.library.address %>"
 salt = "<%= settings.salt %>"

packages/cli/README.md

@@ -16,6 +16,14 @@ Run a package from [the registry](https://usecannon.com/search):
 cannon greeter
 ```
 
+## Run E2E Tests
+
+We use bats as our main testing infrastructure for end-to-end tests. To run the e2e suite, just run the following command in your terminal:
+
+```
+npm run test-e2e
+```
+
 ## Setup to Build
 
 To deploy protocols and create your own packages (i.e. build Cannonfiles), first run the `setup` command:

packages/cli/package.json

@@ -8,6 +8,8 @@
     "watch": "tsc -w",
     "prepublishOnly": "npm run build",
     "test": "jest",
+    "test-e2e": "./test/e2e/helpers/bats/bin/bats test/e2e/e2e-tests.bats",
+    "test-snx-ci": "./test/e2e/helpers/bats/bin/bats test/e2e/synthetix-ci.bats",
     "start": "node bin/cannon.js",
     "docgen": "typedoc"
   },

packages/cli/src/index.ts

@@ -320,7 +320,7 @@ applyCommandsConfig(program.command('build'), commandsConfig.build)
 
     console.log(bold('Building the foundry project...'));
     if (!opts.skipCompile) {
-      const forgeBuildProcess = spawn('forge', ['build'], { cwd: projectDirectory });
+      const forgeBuildProcess = spawn('forge', ['build'], { cwd: projectDirectory, shell: true });
       await new Promise((resolve, reject) => {
         forgeBuildProcess.on('exit', (code) => {
           if (code === 0) {

packages/cli/src/settings.ts

@@ -12,6 +12,7 @@ import {
   DEFAULT_REGISTRY_PROVIDER_URL,
 } from './constants';
 import { filterSettings } from './helpers';
+import { ethers } from 'ethers';
 
 const debug = Debug('cannon:cli:settings');
 
@@ -72,7 +73,7 @@ export type CliSettings = {
   /**
    * URL of etherscan API for verification
    */
-  etherscanApiUrl: string;
+  etherscanApiUrl?: string;
 
   /**
    * Etherscan API Key for verification
@@ -113,9 +114,9 @@ function cannonSettingsSchema(fileSettings: Omit<CliSettings, 'cannonDirectory'>
     CANNON_PROVIDER_URL: z.string().default(fileSettings.providerUrl || 'frame,direct'),
     CANNON_PRIVATE_KEY: z
       .string()
-      .length(64)
-      .optional()
-      .default(fileSettings.privateKey as string),
+      .refine((val) => ethers.utils.isHexString(val, 32), { message: 'Private key is invalid' })
+      .default(fileSettings.privateKey as string)
+      .optional(),
     CANNON_IPFS_URL: z
       .string()
       .url()
@@ -136,7 +137,11 @@ function cannonSettingsSchema(fileSettings: Omit<CliSettings, 'cannonDirectory'>
       .length(42)
       .default(fileSettings.registryAddress || DEFAULT_REGISTRY_ADDRESS),
     CANNON_REGISTRY_PRIORITY: z.enum(['onchain', 'local']).default(fileSettings.registryPriority || 'onchain'),
-    CANNON_ETHERSCAN_API_URL: z.string().url().optional().default(fileSettings.etherscanApiUrl),
+    CANNON_ETHERSCAN_API_URL: z
+      .string()
+      .url()
+      .default(fileSettings.etherscanApiUrl as string)
+      .optional(),
     CANNON_ETHERSCAN_API_KEY: z.string().length(34).optional().default(fileSettings.etherscanApiKey),
     CANNON_QUIET: z.boolean().default(fileSettings.quiet || false),
     TRACE: z.boolean().default(false),
@@ -158,7 +163,7 @@ function _resolveCliSettings(overrides: Partial<CliSettings> = {}): CliSettings
 
   if (!Object.values(fileSettings).length) {
     console.warn(
-      `settings not configured: please create file ${cliSettingsStore} for better performance. See https://usecannon.com/learn/technical-reference#setup for more information.`
+      `settings not configured: please create file ${cliSettingsStore} for better performance. See https://usecannon.com/learn/cli#setup for more information.`
     );
     console.warn(`using default settings (cannon repo, ${DEFAULT_REGISTRY_PROVIDER_URL})`);
   }

packages/cli/test/e2e/config/settings.json

@@ -0,0 +1,10 @@
+{
+  "publishIpfsUrl": "https+ipfs://us-west.repo.usecannon.com", 
+  "providerUrl": "http://localhost:9545",
+  "privateKey": "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80",
+  "registryPriority": "local",
+  "registryProviderUrl": "http://localhost:9545",
+  "etherscanApiUrl": "​https://api.etherscan.io/api",
+  "etherscanApiKey": "A7PNIZCEI2HDRUTDJBWS36XBGIB7Q74YMK"
+ }
+

packages/cli/test/e2e/e2e-tests.bats

@@ -0,0 +1,86 @@
+#!/usr/bin/env bats
+
+# This turns on native debug logs on bash
+# set -x
+
+# File pre-run hook
+setup_file() {
+  load helpers/bats-helpers.sh
+  _setup_file
+
+  # Fork Mainnet to run tests against forked node
+  anvil --fork-url https://ethereum.publicnode.com --port 9545 --silent &
+  export ANVIL_PID="$!"
+}
+
+# File post-run hook
+teardown_file() {
+  load helpers/bats-helpers.sh
+  _teardown_file
+
+  kill -15 "$ANVIL_PID"
+}
+
+# Test pre-hook
+setup() {
+  load helpers/bats-helpers.sh
+  _setup;
+}
+
+# Test post-hook
+teardown() {
+  load helpers/bats-helpers.sh
+  _teardown
+}
+
+@test "Alter - Import contract " {
+  run alter-import-contract.sh
+  echo $output
+  assert_output --partial 'ipfs://QmQZu9RscJYaiwqN2qEhkpEVGTFKPj6K54yV7WnX7CPKAt'
+  assert_success
+}
+
+@test "Alter - Import invoke" {
+  run alter-import-invoke.sh
+  echo $output
+  assert_output --partial 'ipfs://QmQZu9RscJYaiwqN2qEhkpEVGTFKPj6K54yV7WnX7CPKAt'
+  assert_success
+}
+
+@test "Build - Building foundry greeter example" {
+  set_custom_config # Uses custom settings.json
+  run build-foundry.sh
+  echo $output
+  assert_success
+  assert_file_exists "$CANNON_DIRECTORY/tags/greeter-foundry_latest_1-main.txt"
+}
+
+@test "Build - Building hardhat greeter example" {
+  set_custom_config # Uses custom settings.json
+  run build-hardhat.sh
+  echo $output
+  assert_success
+  assert_file_exists "$CANNON_DIRECTORY/tags/greeter_latest_13370-main.txt"
+}
+
+@test "Verify - Verify greeter packages" {
+  set_custom_config # Uses custom settings.json
+  run verify.sh
+  echo $output
+  assert_success
+}
+
+@test "Fetch - Fetch synthetix:latest@main package" {
+  run fetch.sh
+  echo $output
+  assert_output --partial 'Successfully fetched and saved deployment data'
+  assert_success
+  assert_file_exists "$CANNON_DIRECTORY/tags/synthetix_3.3.4_13370-main.txt"
+}
+
+@test "Publish - Publishing package" {
+  set_custom_config # Uses custom settings.json
+  run publish.sh
+  echo $output
+  assert_success
+}

packages/cli/test/e2e/helpers/bats-helpers.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+# DEFAULT BATS FUNCTION OVERRIDES
+
+_setup_file() {
+  export CANNON_REPO_DIR="$(git rev-parse --show-toplevel)"
+  export CANNON="node $CANNON_REPO_DIR/packages/cli/bin/cannon.js"
+
+  # Create temporary directory for tests
+  export WORKDIR="$(mktemp -d)"
+  export CANNON_DIRECTORY="$WORKDIR/cannondir"
+
+  #Creating cannon directory structure
+  mkdir $CANNON_DIRECTORY $CANNON_DIRECTORY/tags/ $CANNON_DIRECTORY/ipfs_cache/ $CANNON_DIRECTORY/metadata_cache/
+
+  # CD into dir so any files created go in the tmp dir
+  cd $CANNON_DIRECTORY  
+}
+
+_setup() {
+  load 'helpers/bats-support/load'
+  load 'helpers/bats-assert/load'
+  load 'helpers/bats-file/load'
+
+  # get the containing directory of this file
+  # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0,
+  # as those will point to the bats executable's location or the preprocessed file respectively
+  DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" >/dev/null 2>&1 && pwd)"
+
+  # make executables in scripts/ visible to PATH so tests can run files without relative path
+  PATH="$DIR/scripts/non-interactive:$DIR/scripts/interactive:$DIR/scripts/:$PATH"
+}
+
+# File post-run hook
+_teardown_file() {
+  if [ -e $WORKDIR]; then
+    rm -rf $WORKDIR
+  fi
+}
+
+# Test post-hook
+_teardown() {
+  if [ -e  "$CANNON_DIRECTORY/settings.json"]; then
+    rm "$CANNON_DIRECTORY/settings.json"
+  fi
+}
+
+
+# CUSTOM FUNCTIONS
+
+# This function is used to log output during test execution
+# Bats by default hides output during each test.
+log() {
+  echo "$@" >&3
+}
+
+# Copy remote network config over to temp dir
+set_custom_config() {
+  touch "$CANNON_DIRECTORY/settings.json"
+  cp "$DIR/config/settings.json" "$CANNON_DIRECTORY/settings.json"
+}
+

packages/cli/test/e2e/scripts/interactive/interact.sh

@@ -0,0 +1 @@
+$CANNON interact synthetix:latest@main --chain-id 1

packages/cli/test/e2e/scripts/interactive/publish-interactive.sh

@@ -0,0 +1 @@
+$CANNON publish greeter-foundry

packages/cli/test/e2e/scripts/interactive/run.sh

@@ -0,0 +1 @@
+$CANNON run synthetix:latest@main --chain-id 5

packages/cli/test/e2e/scripts/non-interactive/alter-import-contract.sh

@@ -0,0 +1 @@
+$CANNON alter synthetix:latest@andromeda --chain-id 84531 import contract.AccountModule 0x009deee64034427b0fe2ab6fb223d79a1b97c59f6374cc910e581c3f5fbf933e

packages/cli/test/e2e/scripts/non-interactive/alter-import-invoke.sh

@@ -0,0 +1 @@
+$CANNON alter synthetix:latest@andromeda --chain-id 84531 import invoke.register_const_one_oracle 0xb864b8ee3b3608e31971f92c3c2fd9ee87af32f1ecc91de1fafe8b8c3347657b

packages/cli/test/e2e/scripts/non-interactive/build-foundry.sh

@@ -0,0 +1 @@
+$CANNON build $CANNON_REPO_DIR/examples/sample-foundry-project/cannonfile.toml --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --chain-id 1 --port 9545

packages/cli/test/e2e/scripts/non-interactive/build-hardhat.sh

@@ -0,0 +1,3 @@
+cd $CANNON_REPO_DIR/examples/sample-hardhat-project;
+npx hardhat cannon:build
+cd -

packages/cli/test/e2e/scripts/non-interactive/downstream-ci-synthetix-deployments.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+git clone --depth=1 https://github.com/Synthetixio/synthetix-deployments
+cd synthetix-deployments
+yarn
+yarn link --all $CANNON_DIRECTORY 
+
+yarn cannon build omnibus-base-mainnet-andromeda.toml \
+  --port 8545 \
+  --dry-run \
+  --upgrade-from synthetix-omnibus:latest@andromeda \
+  --chain-id 8453 \
+  --provider-url https://base.publicnode.com
+

packages/cli/test/e2e/scripts/non-interactive/downstream-ci-synthetix-v3.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+git clone --depth=1 https://github.com/Synthetixio/synthetix-v3
+cd synthetix-v3
+yarn
+yarn link --all $CANNON_DIRECTORY
+
+yarn build

packages/cli/test/e2e/scripts/non-interactive/fetch.sh

@@ -0,0 +1 @@
+$CANNON fetch synthetix:latest@main QmUtELWtepNn1ByaFUF8YAgLMtdvy6GGe2P9ex48EVit4H --chain-id 1

packages/cli/test/e2e/scripts/non-interactive/publish.sh

@@ -0,0 +1 @@
+$CANNON publish greeter-foundry --chain-id 1 --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --skip-confirm

packages/cli/test/e2e/scripts/non-interactive/verify.sh

@@ -0,0 +1 @@
+$CANNON verify greeter-foundry --chain-id 1