diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index 022c18c980..bcfdcdc1fe 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -33,12 +33,9 @@ jobs: HORIZON_INTEGRATION_TESTS_ENABLED: true HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true - PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.3.1-2007.4ede19620.focal - PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21.3.1-2007.4ede19620.focal - PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.5.1 PROTOCOL_22_CORE_DEBIAN_PKG_VERSION: 22.1.0-2194.0241e79f7.focal PROTOCOL_22_CORE_DOCKER_IMG: stellar/stellar-core:22.1.0-2194.0241e79f7.focal - PROTOCOL_22_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:22.0.0-rc3-101 + PROTOCOL_22_STELLAR_RPC_DOCKER_IMG: stellar/stellar-rpc:22.1.1 PGHOST: localhost PGPORT: 5432 PGUSER: postgres @@ -73,11 +70,11 @@ jobs: docker pull "$PROTOCOL_${{ matrix.protocol-version }}_CORE_DOCKER_IMG" echo HORIZON_INTEGRATION_TESTS_DOCKER_IMG="$PROTOCOL_${{ matrix.protocol-version }}_CORE_DOCKER_IMG" >> $GITHUB_ENV - - name: Pull and set Soroban RPC image + - name: Pull and set Stellar RPC image shell: bash run: | - docker pull "$PROTOCOL_${{ matrix.protocol-version }}_SOROBAN_RPC_DOCKER_IMG" - echo HORIZON_INTEGRATION_TESTS_SOROBAN_RPC_DOCKER_IMG="$PROTOCOL_${{ matrix.protocol-version }}_SOROBAN_RPC_DOCKER_IMG" >> $GITHUB_ENV + docker pull "$PROTOCOL_${{ matrix.protocol-version }}_STELLAR_RPC_DOCKER_IMG" + echo HORIZON_INTEGRATION_TESTS_STELLAR_RPC_DOCKER_IMG="$PROTOCOL_${{ matrix.protocol-version }}_STELLAR_RPC_DOCKER_IMG" >> $GITHUB_ENV - name: Install core run: | diff --git a/ingest/change.go b/ingest/change.go index 758a8a11f4..6680faa057 100644 --- a/ingest/change.go +++ b/ingest/change.go @@ -119,7 +119,7 @@ func (c Change) String() string { ) } -func (c Change) ledgerKey() (xdr.LedgerKey, error) { +func (c Change) LedgerKey() (xdr.LedgerKey, error) { if c.Pre != nil { return c.Pre.LedgerKey() } @@ -182,7 +182,7 @@ type sortableChanges struct { func newSortableChanges(changes []Change) sortableChanges { ledgerKeys := make([][]byte, len(changes)) for i, c := range changes { - lk, err := c.ledgerKey() + lk, err := c.LedgerKey() if err != nil { panic(err) } diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg deleted file mode 100644 index 275599bacd..0000000000 --- a/services/horizon/docker/captive-core-integration-tests.cfg +++ /dev/null @@ -1,19 +0,0 @@ -PEER_PORT=11725 -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true -# Lower the TTL of persistent ledger entries -# so that ledger entry extension/restoring becomes testeable -TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="localhost" -QUALITY="MEDIUM" diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg deleted file mode 100644 index 0ce81a7f5c..0000000000 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ /dev/null @@ -1,20 +0,0 @@ -DEPRECATED_SQL_LEDGER_STATE=false -PEER_PORT=11725 -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true -# Lower the TTL of persistent ledger entries -# so that ledger entry extension/restoring becomes testeable -TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="core" -QUALITY="MEDIUM" diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg deleted file mode 100644 index 44820f5933..0000000000 --- a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg +++ /dev/null @@ -1,12 +0,0 @@ -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true -TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true -TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true - -[[VALIDATORS]] -NAME="local_core" -HOME_DOMAIN="core.local" -# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" -PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" -ADDRESS="localhost" -QUALITY="MEDIUM" diff --git a/services/horizon/docker/docker-compose.integration-tests.soroban-rpc.yml b/services/horizon/docker/docker-compose.integration-tests.soroban-rpc.yml deleted file mode 100644 index 940c340a40..0000000000 --- a/services/horizon/docker/docker-compose.integration-tests.soroban-rpc.yml +++ /dev/null @@ -1,20 +0,0 @@ -version: '3' -services: - soroban-rpc: - platform: linux/amd64 - image: ${SOROBAN_RPC_IMAGE:-stellar/soroban-rpc} - depends_on: - - core - restart: on-failure - ports: - - "8080:8080" - environment: - - ENDPOINT=:8080 - - NETWORK_PASSPHRASE=Standalone Network ; February 2017 - - CAPTIVE_CORE_CONFIG_PATH=/captive-core.cfg - - HISTORY_ARCHIVE_URLS=http://core:1570 - - CHECKPOINT_FREQUENCY=8 - - LOG_LEVEL=debug - volumes: - - ./captive-core-integration-tests.soroban-rpc.cfg:/captive-core.cfg - diff --git a/services/horizon/docker/docker-compose.integration-tests.yml b/services/horizon/docker/docker-compose.integration-tests.yml index 630e8b6b60..3282e806d3 100644 --- a/services/horizon/docker/docker-compose.integration-tests.yml +++ b/services/horizon/docker/docker-compose.integration-tests.yml @@ -1,23 +1,11 @@ version: '3' services: - core-postgres: - image: postgres:9.6.17-alpine - restart: on-failure - environment: - - POSTGRES_PASSWORD=mysecretpassword - - POSTGRES_DB=stellar - ports: - - "5641:5641" - command: ["-p", "5641"] core: platform: linux/amd64 # Note: Please keep the image pinned to an immutable tag matching the Captive Core version. # This avoid implicit updates which break compatibility between # the Core container and captive core. image: ${CORE_IMAGE:-stellar/stellar-core:19.13.1-1481.3acf6dd26.focal} - - depends_on: - - core-postgres restart: on-failure environment: - TRACY_NO_INVARIANT_CHECK=1 @@ -29,5 +17,20 @@ services: entrypoint: /usr/bin/env command: /start standalone volumes: - - ./${CORE_CONFIG_FILE:-stellar-core-integration-tests.cfg}:/stellar-core.cfg + - ${CORE_CONFIG_FILE}:/stellar-core.cfg - ./core-start.sh:/start + stellar-rpc: + platform: linux/amd64 + image: ${STELLAR_RPC_IMAGE:-stellar/stellar-rpc} + restart: on-failure + ports: + - "8080:8080" + environment: + - ENDPOINT=:8080 + - NETWORK_PASSPHRASE=Standalone Network ; February 2017 + - CAPTIVE_CORE_CONFIG_PATH=/captive-core.cfg + - HISTORY_ARCHIVE_URLS=http://core:1570 + - CHECKPOINT_FREQUENCY=8 + - LOG_LEVEL=debug + volumes: + - ${CAPTIVE_CORE_CONFIG_FILE}:/captive-core.cfg diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg deleted file mode 100644 index fe23e94e8d..0000000000 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ /dev/null @@ -1,25 +0,0 @@ -DEPRECATED_SQL_LEDGER_STATE=false -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -NETWORK_PASSPHRASE="Standalone Network ; February 2017" - -PEER_PORT=11625 -HTTP_PORT=11626 -PUBLIC_HTTP_PORT=true - -NODE_SEED="SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" - -NODE_IS_VALIDATOR=true -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" - -[QUORUM_SET] -THRESHOLD_PERCENT=100 -VALIDATORS=["GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"] - -[HISTORY.vs] -get="cp history/vs/{0} {1}" -put="cp {0} history/vs/{1}" -mkdir="mkdir -p history/vs/{0}" \ No newline at end of file diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg deleted file mode 100644 index 414ad9c1eb..0000000000 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ /dev/null @@ -1,30 +0,0 @@ -DEPRECATED_SQL_LEDGER_STATE=false -ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - -NETWORK_PASSPHRASE="Standalone Network ; February 2017" - -PEER_PORT=11625 -HTTP_PORT=11626 -PUBLIC_HTTP_PORT=true - -NODE_SEED="SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" - -NODE_IS_VALIDATOR=true -UNSAFE_QUORUM=true -FAILURE_SAFETY=0 - -DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" - -# Lower the TTL of persistent ledger entries -# so that ledger entry extension/restoring becomes testeable -TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 -TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - -[QUORUM_SET] -THRESHOLD_PERCENT=100 -VALIDATORS=["GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"] - -[HISTORY.vs] -get="cp history/vs/{0} {1}" -put="cp {0} history/vs/{1}" -mkdir="mkdir -p history/vs/{0}" diff --git a/services/horizon/internal/integration/change_test.go b/services/horizon/internal/integration/change_test.go index bf2df266eb..0f4a19908d 100644 --- a/services/horizon/internal/integration/change_test.go +++ b/services/horizon/internal/integration/change_test.go @@ -2,17 +2,19 @@ package integration import ( "context" + "io" + "sort" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stellar/go/ingest" "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/keypair" "github.com/stellar/go/services/horizon/internal/test/integration" "github.com/stellar/go/txnbuild" "github.com/stellar/go/xdr" - "github.com/stretchr/testify/assert" - "io" - "sort" - "testing" - "time" ) func TestProtocolUpgradeChanges(t *testing.T) { @@ -20,7 +22,7 @@ func TestProtocolUpgradeChanges(t *testing.T) { itest := integration.NewTest(t, integration.Config{SkipHorizonStart: true}) upgradedLedgerAppx, _ := itest.GetUpgradedLedgerSeqAppx() - waitForLedgerInArchive(t, 15*time.Second, upgradedLedgerAppx) + waitForLedgerInArchive(t, 6*time.Minute, upgradedLedgerAppx) ledgerSeqToLedgers := getLedgers(itest, 2, upgradedLedgerAppx) @@ -61,7 +63,7 @@ func TestOneTxOneOperationChanges(t *testing.T) { tt.NoError(err) ledgerSeq := uint32(txResp.Ledger) - waitForLedgerInArchive(t, 15*time.Second, ledgerSeq) + waitForLedgerInArchive(t, 6*time.Minute, ledgerSeq) ledger := getLedgers(itest, ledgerSeq, ledgerSeq)[ledgerSeq] changes := getChangesFromLedger(itest, ledger) diff --git a/services/horizon/internal/integration/contracts/Cargo.lock b/services/horizon/internal/integration/contracts/Cargo.lock index 9a51ac7a62..5c135d7de5 100644 --- a/services/horizon/internal/integration/contracts/Cargo.lock +++ b/services/horizon/internal/integration/contracts/Cargo.lock @@ -31,9 +31,9 @@ dependencies = [ [[package]] name = "arbitrary" -version = "1.3.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" dependencies = [ "derive_arbitrary", ] @@ -216,14 +216,14 @@ dependencies = [ "num-bigint", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] name = "cc" -version = "1.1.28" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ "shlex", ] @@ -236,9 +236,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -261,9 +261,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -303,12 +303,12 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -335,7 +335,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -359,7 +359,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -370,7 +370,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -412,13 +412,13 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "1.3.2" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -593,9 +593,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "hex" @@ -663,12 +663,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.2", "serde", ] @@ -689,16 +689,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -725,15 +726,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.159" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "log" @@ -771,7 +772,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -843,12 +844,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -862,18 +863,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -948,35 +949,35 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -986,15 +987,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_derive", "serde_json", @@ -1004,14 +1005,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -1066,7 +1067,14 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", +] + +[[package]] +name = "soroban-bulk" +version = "0.0.0" +dependencies = [ + "soroban-sdk", ] [[package]] @@ -1153,7 +1161,7 @@ dependencies = [ "serde", "serde_json", "stellar-xdr", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -1218,7 +1226,7 @@ dependencies = [ "soroban-spec", "soroban-spec-rust", "stellar-xdr", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -1243,7 +1251,7 @@ dependencies = [ "sha2", "soroban-spec", "stellar-xdr", - "syn 2.0.79", + "syn 2.0.94", "thiserror", ] @@ -1348,9 +1356,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "987bc0be1cdea8b10216bd06e2ca407d40b9543468fafd3ddfb02f36e77f71f3" dependencies = [ "proc-macro2", "quote", @@ -1359,29 +1367,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -1400,9 +1408,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -1416,9 +1424,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "version_check" @@ -1434,9 +1442,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -1445,24 +1453,23 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1470,22 +1477,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasmi_arena" @@ -1511,7 +1518,7 @@ version = "0.116.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a58e28b80dd8340cb07b8242ae654756161f6fc8d0038123d679b7b99964fa50" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "semver", ] @@ -1615,7 +1622,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] [[package]] @@ -1635,5 +1642,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.94", ] diff --git a/services/horizon/internal/integration/contracts/Cargo.toml b/services/horizon/internal/integration/contracts/Cargo.toml index 2fcdb8fd5b..a818d0e2b5 100644 --- a/services/horizon/internal/integration/contracts/Cargo.toml +++ b/services/horizon/internal/integration/contracts/Cargo.toml @@ -7,6 +7,7 @@ members = [ "add_u64", "store", "constructor", + "bulk", ] [profile.release-with-logs] diff --git a/services/horizon/internal/integration/contracts/bulk/Cargo.toml b/services/horizon/internal/integration/contracts/bulk/Cargo.toml new file mode 100644 index 0000000000..a48bb7fb3f --- /dev/null +++ b/services/horizon/internal/integration/contracts/bulk/Cargo.toml @@ -0,0 +1,17 @@ +[package] +version = "0.0.0" +name = "soroban-bulk" +authors = ["Stellar Development Foundation "] +license = "Apache-2.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] +doctest = false + +[dependencies] +soroban-sdk = { workspace = true } + +[dev_dependencies] +soroban-sdk = { workspace = true, features = ["testutils"] } \ No newline at end of file diff --git a/services/horizon/internal/integration/contracts/bulk/src/lib.rs b/services/horizon/internal/integration/contracts/bulk/src/lib.rs new file mode 100644 index 0000000000..01bd2182a9 --- /dev/null +++ b/services/horizon/internal/integration/contracts/bulk/src/lib.rs @@ -0,0 +1,19 @@ +#![no_std] +use soroban_sdk::{contract, contractimpl, token, Address, Env, Vec}; + +#[contract] +pub struct Contract; + +#[contractimpl] +impl Contract { + pub fn bulk_transfer(env: Env, sender: Address, contract: Address, recipients: Vec
, amounts: Vec) { + if recipients.len() != amounts.len() { + panic!("number of recipients does not match amounts"); + } + sender.require_auth(); + let client = token::Client::new(&env, &contract); + for (dest, amt) in recipients.iter().zip(amounts.iter()) { + client.transfer(&sender, &dest, &amt); + } + } +} \ No newline at end of file diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 7d85289852..2a60bcf660 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -531,11 +531,6 @@ func TestReingestDB(t *testing.T) { // subprocesses to conflict. itest.StopHorizon() - horizonConfig.CaptiveCoreConfigPath = filepath.Join( - filepath.Dir(horizonConfig.CaptiveCoreConfigPath), - "captive-core-reingest-range-integration-tests.cfg", - ) - var rootCmd = horizoncmd.NewRootCmd() rootCmd.SetArgs(command(t, horizonConfig, "db", "reingest", @@ -909,11 +904,6 @@ func TestFillGaps(t *testing.T) { _, err = historyQ.DeleteRangeAll(context.Background(), oldestLedger, latestLedger) tt.NoError(err) - horizonConfig.CaptiveCoreConfigPath = filepath.Join( - filepath.Dir(horizonConfig.CaptiveCoreConfigPath), - "captive-core-reingest-range-integration-tests.cfg", - ) - rootCmd := horizoncmd.NewRootCmd() rootCmd.SetArgs(command(t, horizonConfig, "db", "fill-gaps", "--parallel-workers=1")) tt.NoError(rootCmd.Execute()) diff --git a/services/horizon/internal/integration/extend_footprint_ttl_test.go b/services/horizon/internal/integration/extend_footprint_ttl_test.go index cc6f947a14..5ccbfa97e8 100644 --- a/services/horizon/internal/integration/extend_footprint_ttl_test.go +++ b/services/horizon/internal/integration/extend_footprint_ttl_test.go @@ -17,7 +17,8 @@ func TestExtendFootprintTtl(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) // establish which account will be contract owner, and load it's current seq diff --git a/services/horizon/internal/integration/invokehostfunction_test.go b/services/horizon/internal/integration/invokehostfunction_test.go index 216bc8607c..28e81d7de4 100644 --- a/services/horizon/internal/integration/invokehostfunction_test.go +++ b/services/horizon/internal/integration/invokehostfunction_test.go @@ -34,7 +34,7 @@ func TestContractInvokeHostFunctionInstallContract(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, }) // establish which account will be contract owner, and load it's current seq @@ -83,7 +83,7 @@ func TestContractInvokeHostFunctionCreateContractByAddress(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, }) // establish which account will be contract owner, and load it's current seq @@ -136,7 +136,8 @@ func TestContractInvokeHostFunctionCreateConstructorContract(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -229,7 +230,7 @@ func TestContractInvokeHostFunctionInvokeStatelessContractFn(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, }) // establish which account will be contract owner @@ -337,7 +338,7 @@ func TestContractInvokeHostFunctionInvokeStatefulContractFn(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, }) // establish which account will be contract owner diff --git a/services/horizon/internal/integration/load_test.go b/services/horizon/internal/integration/load_test.go new file mode 100644 index 0000000000..3e97be721d --- /dev/null +++ b/services/horizon/internal/integration/load_test.go @@ -0,0 +1,481 @@ +package integration + +import ( + "context" + "encoding" + "flag" + "io" + "math/rand" + "os" + "path/filepath" + "sort" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/stellar/go/amount" + "github.com/stellar/go/clients/horizonclient" + "github.com/stellar/go/clients/stellarcore" + "github.com/stellar/go/ingest" + "github.com/stellar/go/keypair" + "github.com/stellar/go/protocols/horizon" + proto "github.com/stellar/go/protocols/stellarcore" + "github.com/stellar/go/services/horizon/internal/test/integration" + "github.com/stellar/go/txnbuild" + "github.com/stellar/go/xdr" +) + +type sorobanTransaction struct { + op *txnbuild.InvokeHostFunction + fee int64 + signer *keypair.Full + sequenceNumber int64 +} + +func TestLoad(t *testing.T) { + var transactionsPerLedger, ledgers, transfersPerTx int + flag.IntVar(&transactionsPerLedger, "transactions-per-ledger", 100, "number of transactions per ledger") + flag.IntVar(&transfersPerTx, "transfers-per-tx", 10, "number of asset transfers for each transaction") + flag.IntVar(&ledgers, "ledgers", 2, "number of ledgers to generate") + flag.Parse() + + if integration.GetCoreMaxSupportedProtocol() < 22 { + t.Skip("This test run does not support less than Protocol 22") + } + + itest := integration.NewTest(t, integration.Config{ + EnableStellarRPC: true, + }) + + maxAccountsPerTransaction := 100 + // transactionsPerLedger should be a multiple of maxAccountsPerTransaction + require.Zero(t, transactionsPerLedger%maxAccountsPerTransaction) + + txErr := itest.CoreClient().UpgradeTxSetSize(context.Background(), uint32(transactionsPerLedger*100), time.Unix(0, 0)) + require.NoError(t, txErr) + + txErr = itest.CoreClient().UpgradeSorobanTxSetSize(context.Background(), uint32(transactionsPerLedger*100), time.Unix(0, 0)) + require.NoError(t, txErr) + + contents, err := os.ReadFile(filepath.Join("testdata", "unlimited-config.xdr")) + require.NoError(t, err) + var configSet xdr.ConfigUpgradeSet + err = xdr.SafeUnmarshalBase64(string(contents), &configSet) + require.NoError(t, err) + + upgradeTransactions, upgradeKey, err := stellarcore.GenSorobanConfigUpgradeTxAndKey(stellarcore.GenSorobanConfig{ + BaseSeqNum: 0, + NetworkPassphrase: integration.StandaloneNetworkPassphrase, + SigningKey: itest.Master(), + StellarCorePath: itest.CoreBinaryPath(), + }, configSet) + require.NoError(t, err) + + for _, transaction := range upgradeTransactions { + var b64 string + b64, err = xdr.MarshalBase64(transaction) + require.NoError(t, err) + var response horizon.Transaction + response, err = itest.Client().SubmitTransactionXDR(b64) + require.NoError(t, err) + require.True(t, response.Successful) + } + + require.NoError(t, + itest.CoreClient().UpgradeSorobanConfig(context.Background(), upgradeKey, time.Unix(0, 0)), + ) + + xlm := xdr.MustNewNativeAsset() + createSAC(itest, xlm) + + bulkContractID, _ := mustCreateAndInstallContract( + itest, + itest.Master(), + "a1", + "soroban_bulk.wasm", + ) + + var signers []*keypair.Full + var accounts []txnbuild.Account + var accountLedgers []uint32 + for i := 0; i < 2*transactionsPerLedger; i += maxAccountsPerTransaction { + keys, curAccounts := itest.CreateAccounts(maxAccountsPerTransaction, "10000000") + var account horizon.Account + account, err = itest.Client().AccountDetail(horizonclient.AccountRequest{AccountID: curAccounts[0].GetAccountID()}) + require.NoError(t, err) + accountLedgers = append(accountLedgers, account.LastModifiedLedger) + + signers = append(signers, keys...) + accounts = append(accounts, curAccounts...) + } + recipients := signers[transactionsPerLedger:] + signers = signers[:transactionsPerLedger] + accounts = accounts[:transactionsPerLedger] + var transactions []sorobanTransaction + var bulkAmounts xdr.ScVec + for i := 0; i < transfersPerTx; i++ { + bulkAmounts = append(bulkAmounts, i128Param(0, uint64(amount.MustParse("1")))) + } + + for i := range signers { + var op *txnbuild.InvokeHostFunction + sender := accounts[i].GetAccountID() + + var bulkRecipients xdr.ScVec + if i%2 == 0 { + for j := i; j < i+transfersPerTx; j++ { + recipient := accountAddressParam(recipients[j%len(recipients)].Address()) + bulkRecipients = append(bulkRecipients, recipient) + } + } else if i%2 == 1 { + for j := 0; j < transfersPerTx; j++ { + var contractID xdr.Hash + _, err = rand.Read(contractID[:]) + require.NoError(t, err) + bulkRecipients = append(bulkRecipients, contractAddressParam(contractID)) + } + } + + op = bulkTransfer(itest, bulkContractID, sender, xlm, &bulkRecipients, &bulkAmounts) + preFlightOp, minFee := itest.PreflightHostFunctions(accounts[i], *op) + preFlightOp.Ext.SorobanData.Resources.ReadBytes *= 10 + preFlightOp.Ext.SorobanData.Resources.WriteBytes *= 10 + preFlightOp.Ext.SorobanData.Resources.Instructions *= 10 + preFlightOp.Ext.SorobanData.ResourceFee *= 10 + minFee *= 10 + var sequenceNumber int64 + sequenceNumber, err = accounts[i].GetSequenceNumber() + require.NoError(t, err) + transactions = append(transactions, sorobanTransaction{ + op: &preFlightOp, + fee: minFee + txnbuild.MinBaseFee, + signer: signers[i], + sequenceNumber: sequenceNumber, + }) + } + + lock := &sync.Mutex{} + ledgerMap := map[int32]int{} + wg := &sync.WaitGroup{} + transactionsPerWorker := 100 + // transactions should be a multiple of transactionsPerWorker + require.Zero(t, len(transactions)%transactionsPerWorker) + for repetitions := 0; repetitions < ledgers; repetitions++ { + for i := 0; i < len(transactions); i += transactionsPerWorker { + subset := transactions[i : i+transactionsPerWorker] + wg.Add(1) + go func() { + defer wg.Done() + txSubWorker( + itest, + subset, + itest.Client(), + itest.CoreClient(), + lock, + ledgerMap, + int64(repetitions), + ) + }() + } + wg.Wait() + } + + start, end := int32(-1), int32(-1) + for ledgerSeq := range ledgerMap { + if start < 0 || start > ledgerSeq { + start = ledgerSeq + } + if end < 0 || ledgerSeq > end { + end = ledgerSeq + } + } + t.Logf("waiting for ledgers [%v, %v] to be in history archive", start, end) + waitForLedgerInArchive(t, 6*time.Minute, uint32(end)) + allLedgers := getLedgers(itest, uint32(start), uint32(end)) + + var sortedLegers []xdr.LedgerCloseMeta + for ledgerSeq := range ledgerMap { + lcm, ok := allLedgers[uint32(ledgerSeq)] + require.True(t, ok) + sortedLegers = append(sortedLegers, lcm) + } + sort.Slice(sortedLegers, func(i, j int) bool { + return sortedLegers[i].LedgerSequence() < sortedLegers[j].LedgerSequence() + }) + + output, err := os.Create(filepath.Join("testdata", "load-test-ledgers.xdr")) + require.NoError(t, err) + _, err = xdr.Marshal(output, sortedLegers) + require.NoError(t, err) + require.NoError(t, output.Close()) + + ledgersForAccounts := getLedgers(itest, accountLedgers[0], accountLedgers[len(accountLedgers)-1]) + var accountLedgerEntries []xdr.LedgerEntry + accountSet := map[string]bool{} + for _, seq := range accountLedgers { + for _, change := range extractChanges(t, []xdr.LedgerCloseMeta{ledgersForAccounts[seq]}) { + if change.Type == xdr.LedgerEntryTypeAccount && change.Post != nil && change.Pre == nil { + account := *change.Post + accountSet[account.Data.MustAccount().AccountId.Address()] = true + accountLedgerEntries = append(accountLedgerEntries, *change.Post) + } + } + } + require.Len(t, accountLedgerEntries, 2*transactionsPerLedger) + + output, err = os.Create(filepath.Join("testdata", "load-test-accounts.xdr")) + require.NoError(t, err) + _, err = xdr.Marshal(output, accountLedgerEntries) + require.NoError(t, err) + require.NoError(t, output.Close()) + + merged := mergeLedgers(t, sortedLegers, transactionsPerLedger) + changes := extractChanges(t, sortedLegers) + for _, change := range changes { + if change.Type != xdr.LedgerEntryTypeAccount { + continue + } + var ledgerKey xdr.LedgerKey + ledgerKey, err = change.LedgerKey() + require.NoError(t, err) + require.True(t, accountSet[ledgerKey.MustAccount().AccountId.Address()]) + } + requireChangesAreEqual(t, changes, extractChanges(t, merged)) + + orignalTransactions := extractTransactions(t, sortedLegers) + mergedTransactions := extractTransactions(t, merged) + require.Equal(t, len(orignalTransactions), len(mergedTransactions)) + for i, original := range orignalTransactions { + requireTransactionsMatch(t, original, mergedTransactions[i]) + } +} + +func bulkTransfer( + itest *integration.Test, + bulkContractID xdr.Hash, + sender string, + asset xdr.Asset, + recipients *xdr.ScVec, + amounts *xdr.ScVec, +) *txnbuild.InvokeHostFunction { + return &txnbuild.InvokeHostFunction{ + HostFunction: xdr.HostFunction{ + Type: xdr.HostFunctionTypeHostFunctionTypeInvokeContract, + InvokeContract: &xdr.InvokeContractArgs{ + ContractAddress: contractIDParam(bulkContractID), + FunctionName: "bulk_transfer", + Args: xdr.ScVec{ + accountAddressParam(sender), + contractAddressParam(stellarAssetContractID(itest, asset)), + xdr.ScVal{Type: xdr.ScValTypeScvVec, Vec: &recipients}, + xdr.ScVal{Type: xdr.ScValTypeScvVec, Vec: &amounts}, + }, + }, + }, + SourceAccount: sender, + } +} + +func extractChanges(t *testing.T, ledgers []xdr.LedgerCloseMeta) []ingest.Change { + var changes []ingest.Change + for _, ledger := range ledgers { + reader, err := ingest.NewLedgerChangeReaderFromLedgerCloseMeta(integration.StandaloneNetworkPassphrase, ledger) + require.NoError(t, err) + for { + var change ingest.Change + change, err = reader.Read() + if err == io.EOF { + break + } + require.NoError(t, err) + changes = append(changes, change) + } + } + return changes +} + +func extractTransactions(t *testing.T, ledgers []xdr.LedgerCloseMeta) []ingest.LedgerTransaction { + var transactions []ingest.LedgerTransaction + for _, ledger := range ledgers { + txReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(integration.StandaloneNetworkPassphrase, ledger) + require.NoError(t, err) + for { + var tx ingest.LedgerTransaction + tx, err = txReader.Read() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + transactions = append(transactions, tx) + } + } + return transactions +} + +func groupChangesByLedgerKey(t *testing.T, changes []ingest.Change) map[string][]ingest.Change { + byLedgerKey := map[string][]ingest.Change{} + for _, change := range changes { + key, err := change.LedgerKey() + require.NoError(t, err) + keyB64, err := key.MarshalBinaryBase64() + require.NoError(t, err) + byLedgerKey[keyB64] = append(byLedgerKey[keyB64], change) + } + return byLedgerKey +} + +func requireChangesAreEqual(t *testing.T, a, b []ingest.Change) { + aByLedgerKey := groupChangesByLedgerKey(t, a) + bByLedgerKey := groupChangesByLedgerKey(t, b) + + for key, aChanges := range aByLedgerKey { + bChanges := bByLedgerKey[key] + require.Equal(t, len(aChanges), len(bChanges)) + for i, aChange := range aChanges { + bChange := bChanges[i] + require.Equal(t, aChange.Reason, bChange.Reason) + require.Equal(t, aChange.Type, bChange.Type) + if aChange.Pre == nil { + require.Nil(t, bChange.Pre) + } else { + requireXDREquals(t, aChange.Pre, bChange.Pre) + } + if aChange.Post == nil { + require.Nil(t, bChange.Post) + } else { + requireXDREquals(t, aChange.Post, bChange.Post) + } + } + } +} + +func requireTransactionsMatch(t *testing.T, a, b ingest.LedgerTransaction) { + requireXDREquals(t, a.Hash, b.Hash) + requireXDREquals(t, a.UnsafeMeta, b.UnsafeMeta) + requireXDREquals(t, a.Result, b.Result) + requireXDREquals(t, a.Envelope, b.Envelope) + requireXDREquals(t, a.FeeChanges, b.FeeChanges) + require.Equal(t, a.LedgerVersion, b.LedgerVersion) +} + +func requireXDREquals(t *testing.T, a, b encoding.BinaryMarshaler) { + serialized, err := a.MarshalBinary() + require.NoError(t, err) + otherSerialized, err := b.MarshalBinary() + require.NoError(t, err) + require.Equal(t, serialized, otherSerialized) +} + +func txSubWorker( + itest *integration.Test, + subset []sorobanTransaction, + horizonClient *horizonclient.Client, + coreClient *stellarcore.Client, + ledgerLock *sync.Mutex, + ledgerMap map[int32]int, + sequenceOffset int64, +) { + var total time.Duration + pending := map[string]bool{} + for _, tx := range subset { + account := txnbuild.NewSimpleAccount(tx.signer.Address(), tx.sequenceNumber+sequenceOffset) + tx, err := itest.CreateSignedTransactionFromOpsWithFee(&account, []*keypair.Full{tx.signer}, tx.fee, tx.op) + require.NoError(itest.CurrentTest(), err) + + hash, err := tx.HashHex(integration.StandaloneNetworkPassphrase) + require.NoError(itest.CurrentTest(), err) + b64Tx, err := tx.Base64() + require.NoError(itest.CurrentTest(), err) + start := time.Now() + resp, err := coreClient.SubmitTransaction(context.Background(), b64Tx) + elapsed := time.Since(start) + require.NoError(itest.CurrentTest(), err) + require.Empty(itest.CurrentTest(), resp.Exception) + require.False(itest.CurrentTest(), resp.IsException()) + require.Equal(itest.CurrentTest(), proto.TXStatusPending, resp.Status) + pending[hash] = true + total += elapsed + } + avg := total / time.Duration(len(subset)) + itest.CurrentTest().Logf("avg %v total %v", avg, total) + + start := time.Now() + waitForTransactions(itest.CurrentTest(), horizonClient, pending, ledgerLock, ledgerMap) + itest.CurrentTest().Logf("wait duration %v", time.Since(start)) + +} + +func waitForTransactions( + t *testing.T, + client *horizonclient.Client, + pending map[string]bool, + ledgerLock *sync.Mutex, + ledgerMap map[int32]int, +) { + require.Eventually(t, func() bool { + for hash := range pending { + resp, err := client.TransactionDetail(hash) + if err == nil { + delete(pending, hash) + require.True(t, resp.Successful) + ledgerLock.Lock() + ledgerMap[resp.Ledger]++ + ledgerLock.Unlock() + continue + } + if horizonclient.IsNotFoundError(err) { + continue + } else { + require.NoError(t, err) + } + } + return len(pending) == 0 + }, time.Second*90, time.Millisecond*100) +} + +func mergeLedgers(t *testing.T, ledgers []xdr.LedgerCloseMeta, transactionsPerLedger int) []xdr.LedgerCloseMeta { + var merged []xdr.LedgerCloseMeta + if len(ledgers) == 0 { + return merged + } + var cur xdr.LedgerCloseMeta + var curCount int + for _, ledger := range ledgers { + transactionCount := ledger.CountTransactions() + require.Empty(t, ledger.V1.EvictedTemporaryLedgerKeys) + require.Empty(t, ledger.V1.EvictedPersistentLedgerEntries) + require.Empty(t, ledger.V1.UpgradesProcessing) + if transactionCount == 0 { + continue + } + + if curCount == 0 { + cur = copyLedger(t, ledger) + cur.V1.TxProcessing = nil + } else { + cur.V1.TxSet.V1TxSet.Phases = append(cur.V1.TxSet.V1TxSet.Phases, ledger.V1.TxSet.V1TxSet.Phases...) + } + + require.LessOrEqual(t, curCount+transactionCount, transactionsPerLedger) + cur.V1.TxProcessing = append(cur.V1.TxProcessing, ledger.V1.TxProcessing...) + curCount += transactionCount + if curCount == transactionsPerLedger { + merged = append(merged, cur) + curCount = 0 + } + } + require.Zero(t, curCount) + return merged +} + +func copyLedger(t *testing.T, src xdr.LedgerCloseMeta) xdr.LedgerCloseMeta { + var dst xdr.LedgerCloseMeta + serialized, err := src.MarshalBinary() + require.NoError(t, err) + require.NoError(t, dst.UnmarshalBinary(serialized)) + return dst +} diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 7ded1f9a88..25b2871db5 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -27,11 +27,6 @@ import ( "github.com/stretchr/testify/assert" ) -var defaultCaptiveCoreParameters = map[string]string{ - horizon.StellarCoreBinaryPathName: os.Getenv("CAPTIVE_CORE_BIN"), - horizon.StellarCoreURLFlagName: "", -} - var networkParamArgs = map[string]string{ horizon.CaptiveCoreConfigPathName: "", horizon.CaptiveCoreHTTPPortFlagName: "", @@ -41,29 +36,6 @@ var networkParamArgs = map[string]string{ horizon.NetworkPassphraseFlagName: "", } -var ( - CaptiveCoreConfigErrMsg = "error generating captive core configuration: invalid config: " -) - -// Ensures that BUCKET_DIR_PATH is not an allowed value for Captive Core. -func TestBucketDirDisallowed(t *testing.T) { - config := `BUCKET_DIR_PATH="/tmp" - ` + integration.SimpleCaptiveCoreToml - - confName, _, cleanup := integration.CreateCaptiveCoreConfig(config) - defer cleanup() - testConfig := integration.GetTestConfig() - testConfig.HorizonIngestParameters = map[string]string{ - horizon.CaptiveCoreConfigPathName: confName, - horizon.StellarCoreBinaryPathName: os.Getenv("CAPTIVE_CORE_BIN"), - } - test := integration.NewTest(t, *testConfig) - err := test.StartHorizon(true) - assert.Equal(t, err.Error(), integration.HorizonInitErrStr+": error generating captive core configuration:"+ - " invalid captive core toml file: could not unmarshal captive core toml: setting BUCKET_DIR_PATH is disallowed"+ - " for Captive Core, use CAPTIVE_CORE_STORAGE_PATH instead") -} - func TestEnvironmentPreserved(t *testing.T) { // Who tests the tests? This test. // @@ -234,28 +206,9 @@ func TestNetworkEnvironmentVariable(t *testing.T) { // Ensures that the filesystem ends up in the correct state with Captive Core. func TestCaptiveCoreConfigFilesystemState(t *testing.T) { - confName, storagePath, cleanup := integration.CreateCaptiveCoreConfig(integration.SimpleCaptiveCoreToml) - defer cleanup() - - localParams := integration.MergeMaps(defaultCaptiveCoreParameters, map[string]string{ - "captive-core-storage-path": storagePath, - horizon.CaptiveCoreConfigPathName: confName, - }) - testConfig := integration.GetTestConfig() - testConfig.HorizonIngestParameters = localParams - test := integration.NewTest(t, *testConfig) - - err := test.StartHorizon(true) - assert.NoError(t, err) - test.WaitForHorizonIngest() - - t.Run("disk state", func(t *testing.T) { - validateCaptiveCoreDiskState(test, storagePath) - }) - - t.Run("no bucket dir", func(t *testing.T) { - validateNoBucketDirPath(test, storagePath) - }) + test := integration.NewTest(t, integration.Config{}) + validateCaptiveCoreDiskState(test, test.CaptiveCoreStoragePath()) + validateNoBucketDirPath(test, test.CaptiveCoreStoragePath()) } func TestMaxAssetsForPathRequests(t *testing.T) { diff --git a/services/horizon/internal/integration/sac_test.go b/services/horizon/internal/integration/sac_test.go index 4a9db284bf..be06fe32f3 100644 --- a/services/horizon/internal/integration/sac_test.go +++ b/services/horizon/internal/integration/sac_test.go @@ -41,7 +41,8 @@ func TestContractMintToAccount(t *testing.T) { itest := integration.NewTest(t, integration.Config{ HorizonEnvironment: map[string]string{"INGEST_DISABLE_STATE_VERIFICATION": "true", "CONNECTION_TIMEOUT": "360000"}, - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -144,7 +145,8 @@ func TestContractMintToContract(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -223,13 +225,14 @@ func TestExpirationAndRestoration(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, HorizonIngestParameters: map[string]string{ // disable state verification because we will insert // a fake asset contract in the horizon db and we don't // want state verification to detect this "ingest-disable-state-verification": "true", }, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -501,7 +504,8 @@ func TestContractTransferBetweenAccounts(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -575,7 +579,8 @@ func TestContractTransferBetweenAccountAndContract(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -695,7 +700,8 @@ func TestContractTransferBetweenContracts(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -776,7 +782,8 @@ func TestContractBurnFromAccount(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -851,7 +858,8 @@ func TestContractBurnFromContract(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) issuer := itest.Master().Address() @@ -918,7 +926,8 @@ func TestContractClawbackFromAccount(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) // Give the master account the revocable flag (needed to set the clawback flag) @@ -995,7 +1004,8 @@ func TestContractClawbackFromContract(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, + QuickExpiration: true, }) // Give the master account the revocable flag (needed to set the clawback flag) diff --git a/services/horizon/internal/integration/testdata/soroban_bulk.wasm b/services/horizon/internal/integration/testdata/soroban_bulk.wasm new file mode 100755 index 0000000000..f413b43a53 Binary files /dev/null and b/services/horizon/internal/integration/testdata/soroban_bulk.wasm differ diff --git a/services/horizon/internal/integration/testdata/unlimited-config.xdr b/services/horizon/internal/integration/testdata/unlimited-config.xdr new file mode 100644 index 0000000000..14bd0c4924 --- /dev/null +++ b/services/horizon/internal/integration/testdata/unlimited-config.xdr @@ -0,0 +1 @@ +AAAACwAAAAD/////AAAAAR//////////H/////////8AAAAAAAAAZP////8AAAAC//////////////////////////////////////////8AAAAAAAAD6AAAAAAAAAu4AAAAAAAAA+ggAAAAAAAAMAAAAAAAAAPoAAAAAAA9CQAAAAPoAAAAAwAAAAAAABOIAAAABP////8AAAAAAAABLAAAAAX//////////wAAAAAAAAH0AAAABgAAAEYAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAABsgAAAAAAAAAQAAAAAAAAAAAAAAAqAAAAAAAAABAAAAAAAAAAAAAAACwAAAAAAAAAEAAAAAAAAAAAAAABJwAAAAAAAAAAAAAAAAAAAAAAAAA8AAAAAAAAAAAAAAAAAAAAAAAAAN0AAAAAAAAAGgAAAAAAAAAAAAABSwAAAAAAABERAAAAAAAAAAAAAA40AAAAAAAAG2UAAAAAAAAAAAAAnUAAAAAAAAAAAAAAAAAAAAAAAAXCzwAAAAAAAA/bAAAAAAAAAAAABl7KAAAAAAAAspAAAAAAAAAAAAAAoLYAAAAAAAACegAAAAAAAAAAAAAHmQAAAAAAAAAAAAAAAAAAAAAAABlRAAAAAAAAFzcAAAAAAAAAAAAAAscAAAAAAAAAAAAAAAAAAAAAACNSNAAAAAAAAAAAAAAAAAAAAAAAABBQAAAAAAAAAAAAAAAAAAAAAAAAEmwAAAAAAAAAAAAAAAAAAAAAAAASSAAAAAAAAAAAAAAAAAAAAAAAABCgAAAAAAAAAAAAAAAAAAAAAAAAA3QAAAAAAAAAAAAAAAAAAAAAAAAEIwAAAAAAAAH2AAAAAAAAAAAAAR11AAAAAAAAY0IAAAAAAAAAAAAAAAAAAAAAAAhAUAAAAAAAAAAAAAAAAAAAAAAAArDrAAAAAAAAAAAAAAAAAAAAAAAAdSUAAAAAAAAAAAAAAAAAAAAAABAySQAAAAAAAAAAAAAAAAAAAAAAA58YAAAAAAAAAAAAAAAAAAAAAAAFAxwAAAAAAAAAAAAAAAAAAAAAAAq1lQAAAAAAAAAAAAAAAAAAAAAABo1HAAAAAAAAAAAAAAAAAAAAAAAAABwAAAAAAAAAAAAAqBYAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB2EAAAAAAAAAAAAAAAAAAAAAAAAKdcAAAAAAAAAAAAAAAAAAAAAAAAM5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWf4AAAAAAAAAAAAAAAAAAAAAAACl+AAAAAAAAAAAAAAAAAAAAAAADKYuAAAAAAAAAAAAAAAAAAAAAAAEiIwAAAAAAAAAAAAAAAAAAAAAAAAADgAAAAAAAAAAAAAHWgAAAAAAAAAAAAAAAAAAAAAALcpKAAAAAAAAAAAAAAAAAAAAAAAAApUAAAAAAAAAAAAAAAAAAAAAAAAD2QAAAAAAAAAAAAAAAAAAAAAAAAeOAAAAAAAAAAAAAAAAAAAAAAALJY4AAAAAAAAAAAAAAAAAAAAAAAAXIQAAAAAAAAAAAAAAAAAAAAAAECQeAAAAAAAAAAAAAAAAAAAAAAABaeIAAAAAAAAAAAAAAAAAAAAAAAGJhgAAAAAAAAAAAAAAAAAAAAAAAB4JAAAAAAAAAAAAAAAAAAAAAAAlhWkAAAAAAAAAAAAAAAAAAAAAACUHYgAAAAAFvulnAAAAAAAAAAAAF4WyAAAAAAAAAAAAAAAAAAAAAAAw/7cAAAAAAAAaOQAAAAAAAAAAAABidwAAAAAAAAAAAAAAAAAAAAAAeCLDAAAAAAAAAAAAAAAAAAAAAAB6noAAAAAAEnUmBwAAAAAAAAAAACTt6gAAAAAAAAAAAAAAAAAAAAAAa5VEAAAAAAAAGo0AAAAAAAAAAAChHeQAAAAAJbixDwAAAAAAAAAAAAAHygAAAAAAAAAAAAAAAAAAAAAAAASDAAAAAAAAAAAAAAAAAAAAAAAAAEoAAAAAAAAAAAAAAAAAAAAAAAABTAAAAAAAAAAAAAAAAAAAAAAAAAKzAAAAAAABIz4AAAAAAAAAAAAAil0AAAAAAAAAAAAAAAj/////AAAACf////8AAAAKAC92AAAAABAAAdiAAAAAAAAILIAAAAAAAFG9AAAAAGQAAAAeAAAAQAABhqAAAAAGAAAAC/////8= \ No newline at end of file diff --git a/services/horizon/internal/integration/transaction_test.go b/services/horizon/internal/integration/transaction_test.go index 8c3fa369fb..24b560b3d7 100644 --- a/services/horizon/internal/integration/transaction_test.go +++ b/services/horizon/internal/integration/transaction_test.go @@ -15,7 +15,7 @@ import ( func TestP19MetaTransaction(t *testing.T) { itest := integration.NewTest(t, integration.Config{ ProtocolVersion: 19, - EnableSorobanRPC: false, + EnableStellarRPC: false, }) masterAccount, err := itest.Client().AccountDetail(horizonclient.AccountRequest{ @@ -46,7 +46,7 @@ func TestP19MetaDisabledTransaction(t *testing.T) { itest := integration.NewTest(t, integration.Config{ ProtocolVersion: 19, HorizonEnvironment: map[string]string{"SKIP_TXMETA": "TRUE"}, - EnableSorobanRPC: false, + EnableStellarRPC: false, }) masterAccount, err := itest.Client().AccountDetail(horizonclient.AccountRequest{ @@ -72,7 +72,7 @@ func TestP20MetaTransaction(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, }) // establish which account will be contract owner, and load it's current seq @@ -102,7 +102,7 @@ func TestP20MetaDisabledTransaction(t *testing.T) { itest := integration.NewTest(t, integration.Config{ HorizonEnvironment: map[string]string{"SKIP_TXMETA": "TRUE"}, - EnableSorobanRPC: true, + EnableStellarRPC: true, }) // establish which account will be contract owner, and load it's current seq diff --git a/services/horizon/internal/integration/txsub_async_test.go b/services/horizon/internal/integration/txsub_async_test.go index 8310be1dba..1701702775 100644 --- a/services/horizon/internal/integration/txsub_async_test.go +++ b/services/horizon/internal/integration/txsub_async_test.go @@ -135,10 +135,11 @@ func TestAsyncTxSub_SubmissionTryAgainLater(t *testing.T) { func TestAsyncTxSub_TransactionMalformed(t *testing.T) { itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, HorizonEnvironment: map[string]string{ "MAX_HTTP_REQUEST_SIZE": "1800", }, + QuickExpiration: true, }) master := itest.Master() diff --git a/services/horizon/internal/integration/txsub_test.go b/services/horizon/internal/integration/txsub_test.go index e253522893..04c2eb17f0 100644 --- a/services/horizon/internal/integration/txsub_test.go +++ b/services/horizon/internal/integration/txsub_test.go @@ -60,10 +60,11 @@ func TestTxSubLimitsBodySize(t *testing.T) { } itest := integration.NewTest(t, integration.Config{ - EnableSorobanRPC: true, + EnableStellarRPC: true, HorizonEnvironment: map[string]string{ "MAX_HTTP_REQUEST_SIZE": "1800", }, + QuickExpiration: true, }) // establish which account will be contract owner, and load it's current seq diff --git a/services/horizon/internal/test/integration/core_config.go b/services/horizon/internal/test/integration/core_config.go new file mode 100644 index 0000000000..2999ec1efe --- /dev/null +++ b/services/horizon/internal/test/integration/core_config.go @@ -0,0 +1,69 @@ +package integration + +type validatorCoreConfigTemplatePrams struct { + Accelerate bool + NetworkPassphrase string + TestingMinimumPersistentEntryLifetime int + TestingSorobanHighLimitOverride bool +} + +type captiveCoreConfigTemplatePrams struct { + validatorCoreConfigTemplatePrams + ValidatorAddress string +} + +const validatorCoreConfigTemplate = ` +DEPRECATED_SQL_LEDGER_STATE=false +ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING={{ .Accelerate }} + +NETWORK_PASSPHRASE="{{ .NetworkPassphrase }}" + +TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME={{ .TestingMinimumPersistentEntryLifetime }} +TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE={{ .TestingSorobanHighLimitOverride }} + +PEER_PORT=11625 +HTTP_PORT=11626 +PUBLIC_HTTP_PORT=true + +NODE_SEED="SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" + +NODE_IS_VALIDATOR=true +UNSAFE_QUORUM=true +FAILURE_SAFETY=0 + +BUCKETLIST_DB_INDEX_PAGE_SIZE_EXPONENT = 12 +DATABASE = "sqlite3://stellar.db" + +[QUORUM_SET] +THRESHOLD_PERCENT=100 +VALIDATORS=["GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"] + +[HISTORY.vs] +get="cp history/vs/{0} {1}" +put="cp {0} history/vs/{1}" +mkdir="mkdir -p history/vs/{0}" +` + +const captiveCoreConfigTemplate = ` +DEPRECATED_SQL_LEDGER_STATE=false +ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING={{ .Accelerate }} + +NETWORK_PASSPHRASE="{{ .NetworkPassphrase }}" + +TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME={{ .TestingMinimumPersistentEntryLifetime }} +TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE={{ .TestingSorobanHighLimitOverride }} +ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true + +PEER_PORT=11725 + +UNSAFE_QUORUM=true +FAILURE_SAFETY=0 + +[[VALIDATORS]] +NAME="local_core" +HOME_DOMAIN="core.local" +# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK" +PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" +ADDRESS="{{ .ValidatorAddress }}" +QUALITY="MEDIUM" +` diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index f90eece01f..99aa8af719 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -4,8 +4,6 @@ package integration import ( "context" "fmt" - "github.com/stellar/go/historyarchive" - "github.com/stellar/go/ingest/ledgerbackend" "io/ioutil" "os" "os/exec" @@ -16,8 +14,12 @@ import ( "sync" "syscall" "testing" + "text/template" "time" + "github.com/stellar/go/historyarchive" + "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/services/horizon/internal/test" "github.com/2opremio/pretty" @@ -47,33 +49,12 @@ const ( AdminPort = 6060 StellarCorePort = 11626 HistoryArchivePort = 1570 - SorobanRPCPort = 8080 + StellarRPCPort = 8080 HistoryArchiveUrl = "http://localhost:1570" CheckpointFrequency = 8 ) const ( - SimpleCaptiveCoreToml = ` - PEER_PORT=11725 - ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true - NETWORK_PASSPHRASE = "Standalone Network ; February 2017" - UNSAFE_QUORUM=true - FAILURE_SAFETY=0 - RUN_STANDALONE=false - - # Lower the TTL of persistent ledger entries - # so that ledger entry extension/restoring becomes testeable - # These 2 settings need to be present in both places - stellar-core-integration-tests.cfg and here - TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 - TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true - - [[VALIDATORS]] - NAME="local_core" - HOME_DOMAIN="core.local" - PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" - ADDRESS="localhost" - QUALITY="MEDIUM" -` StellarCoreURL = "http://localhost:11626" ) @@ -81,12 +62,13 @@ const HorizonInitErrStr = "cannot initialize Horizon" type Config struct { ProtocolVersion uint32 - EnableSorobanRPC bool + EnableStellarRPC bool + LogContainers bool SkipCoreContainerCreation bool - SkipCoreContainerDeletion bool // This flag is helpful to debug CoreDockerImage string - SorobanRPCDockerImage string + StellarRPCDockerImage string SkipProtocolUpgrade bool + QuickExpiration bool // Weird naming here because bools default to false, but we want to start // Horizon by default. @@ -110,13 +92,14 @@ type CaptiveConfig struct { binaryPath string configPath string storagePath string - useDB bool } type Test struct { t *testing.T - composePath string + composePath string + validatorConfPath string + rpcCoreConfPath string config Config coreConfig CaptiveConfig @@ -175,6 +158,16 @@ func NewTest(t *testing.T, config Config) *Test { config.ProtocolVersion = maxSupportedCoreProtocolFromEnv } } + validatorParams := validatorCoreConfigTemplatePrams{ + Accelerate: CheckpointFrequency < historyarchive.DefaultCheckpointFrequency, + NetworkPassphrase: StandaloneNetworkPassphrase, + TestingMinimumPersistentEntryLifetime: 65536, + TestingSorobanHighLimitOverride: false, + } + if config.QuickExpiration { + validatorParams.TestingSorobanHighLimitOverride = true + validatorParams.TestingMinimumPersistentEntryLifetime = 10 + } var i *Test if !config.SkipCoreContainerCreation { composePath := findDockerComposePath() @@ -185,9 +178,13 @@ func NewTest(t *testing.T, config Config) *Test { passPhrase: StandaloneNetworkPassphrase, environment: test.NewEnvironmentManager(), } - i.configureCaptiveCore() + i.validatorConfPath = i.createCoreValidatorConf(validatorParams) + i.rpcCoreConfPath = i.createCaptiveCoreConf(captiveCoreConfigTemplatePrams{ + validatorCoreConfigTemplatePrams: validatorParams, + ValidatorAddress: "core", + }) // Only run Stellar Core container and its dependencies. - i.runComposeCommand("up", "--detach", "--quiet-pull", "--no-color", "core") + i.startCoreValidator() } else { i = &Test{ t: t, @@ -195,14 +192,15 @@ func NewTest(t *testing.T, config Config) *Test { environment: test.NewEnvironmentManager(), } } + i.configureCaptiveCore(validatorParams) i.prepareShutdownHandlers() i.coreClient = &stellarcore.Client{URL: "http://localhost:" + strconv.Itoa(StellarCorePort)} if !config.SkipCoreContainerCreation { i.waitForCore() - if i.config.EnableSorobanRPC { - i.runComposeCommand("up", "--detach", "--quiet-pull", "--no-color", "soroban-rpc") - i.waitForSorobanRPC() + if i.config.EnableStellarRPC { + i.startRPC() + i.waitForStellarRPC() } } @@ -217,12 +215,13 @@ func NewTest(t *testing.T, config Config) *Test { return i } -func (i *Test) configureCaptiveCore() { - composePath := findDockerComposePath() +func (i *Test) configureCaptiveCore(validatorParams validatorCoreConfigTemplatePrams) { i.coreConfig.binaryPath = os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN") - i.coreConfig.configPath = filepath.Join(composePath, "captive-core-integration-tests.cfg") + i.coreConfig.configPath = i.createCaptiveCoreConf(captiveCoreConfigTemplatePrams{ + validatorCoreConfigTemplatePrams: validatorParams, + ValidatorAddress: "localhost", + }) i.coreConfig.storagePath = i.CurrentTest().TempDir() - i.coreConfig.useDB = true if value := i.getIngestParameter( horizon.StellarCoreBinaryPathName, @@ -248,57 +247,121 @@ func (i *Test) getIngestParameter(argName, envName string) string { return "" } +func (i *Test) createCoreValidatorConf(params validatorCoreConfigTemplatePrams) string { + tomlFile, err := os.CreateTemp("", "stellar-core-integration-test-*.toml") + require.NoError(i.t, err) + + tmpl, err := template.New("core-validator").Parse(validatorCoreConfigTemplate) + require.NoError(i.t, err) + err = tmpl.Execute(tomlFile, params) + require.NoError(i.t, err) + + require.NoError(i.t, tomlFile.Close()) + + i.t.Cleanup(func() { + require.NoError(i.t, os.Remove(tomlFile.Name())) + }) + return tomlFile.Name() +} + +func (i *Test) createCaptiveCoreConf(params captiveCoreConfigTemplatePrams) string { + tomlFile, err := os.CreateTemp("", "captive-core-integration-test-*.toml") + require.NoError(i.t, err) + + tmpl, err := template.New("captive-core").Parse(captiveCoreConfigTemplate) + require.NoError(i.t, err) + err = tmpl.Execute(tomlFile, params) + require.NoError(i.t, err) + + require.NoError(i.t, tomlFile.Close()) + + i.t.Cleanup(func() { + require.NoError(i.t, os.Remove(tomlFile.Name())) + }) + return tomlFile.Name() +} + // Runs a docker-compose command applied to the above configs -func (i *Test) runComposeCommand(args ...string) { - integrationYaml := filepath.Join(i.composePath, "docker-compose.integration-tests.yml") - integrationSorobanRPCYaml := filepath.Join(i.composePath, "docker-compose.integration-tests.soroban-rpc.yml") +func (i *Test) runComposeCommand(envVars []string, args ...string) { + cmdline := append( + []string{ + "compose", + "-f", + filepath.Join(i.composePath, "docker-compose.integration-tests.yml"), + }, + args..., + ) + cmd := exec.Command("docker", cmdline...) + + cmd.Env = append( + envVars, + fmt.Sprintf("CAPTIVE_CORE_CONFIG_FILE=%s", i.rpcCoreConfPath), + fmt.Sprintf("CORE_CONFIG_FILE=%s", i.validatorConfPath), + ) - cmdline := args - if i.config.EnableSorobanRPC { - cmdline = append([]string{"-f", integrationSorobanRPCYaml}, cmdline...) + i.t.Log("Running", cmd.Args) + out, innerErr := cmd.Output() + if len(out) > 0 { + fmt.Printf("stdout:\n%s\n", string(out)) } - cmdline = append([]string{"-f", integrationYaml}, cmdline...) - cmdline = append([]string{"compose"}, cmdline...) - cmd := exec.Command("docker", cmdline...) - coreImageOverride := "" + if exitErr, ok := innerErr.(*exec.ExitError); ok { + fmt.Printf("stderr:\n%s\n", string(exitErr.Stderr)) + } + + if innerErr != nil { + i.t.Fatalf("Compose command failed: %v", innerErr) + } +} + +func (i *Test) startCoreValidator() { + var envVars []string + var coreImageOverride string + if i.config.CoreDockerImage != "" { coreImageOverride = i.config.CoreDockerImage } else if img := os.Getenv("HORIZON_INTEGRATION_TESTS_DOCKER_IMG"); img != "" { coreImageOverride = img } - - cmd.Env = os.Environ() if coreImageOverride != "" { - cmd.Env = append( - cmd.Environ(), + envVars = append( + envVars, fmt.Sprintf("CORE_IMAGE=%s", coreImageOverride), ) } - sorobanRPCOverride := "" - if i.config.SorobanRPCDockerImage != "" { - sorobanRPCOverride = i.config.CoreDockerImage - } else if img := os.Getenv("HORIZON_INTEGRATION_TESTS_SOROBAN_RPC_DOCKER_IMG"); img != "" { - sorobanRPCOverride = img + + i.runComposeCommand(envVars, "up", "--detach", "--quiet-pull", "--no-color", "core") +} + +func (i *Test) startRPC() { + var envVars []string + var stellarRPCOverride string + + if i.config.StellarRPCDockerImage != "" { + stellarRPCOverride = i.config.CoreDockerImage + } else if img := os.Getenv("HORIZON_INTEGRATION_TESTS_STELLAR_RPC_DOCKER_IMG"); img != "" { + stellarRPCOverride = img } - if sorobanRPCOverride != "" { - cmd.Env = append( - cmd.Environ(), - fmt.Sprintf("SOROBAN_RPC_IMAGE=%s", sorobanRPCOverride), + if stellarRPCOverride != "" { + envVars = append( + envVars, + fmt.Sprintf("STELLAR_RPC_IMAGE=%s", stellarRPCOverride), ) } - i.t.Log("Running", cmd.Args) - out, innerErr := cmd.Output() - if len(out) > 0 { - fmt.Printf("stdout:\n%s\n", string(out)) - } - if exitErr, ok := innerErr.(*exec.ExitError); ok { - fmt.Printf("stderr:\n%s\n", string(exitErr.Stderr)) - } + i.runComposeCommand(envVars, "up", "--detach", "--quiet-pull", "--no-color", "stellar-rpc") +} - if innerErr != nil { - i.t.Fatalf("Compose command failed: %v", innerErr) - } +func (i *Test) removeContainers(containers ...string) { + i.runComposeCommand( + nil, + append( + []string{ + "rm", + "-fvs", + }, + containers..., + )..., + ) } func (i *Test) prepareShutdownHandlers() { @@ -311,16 +374,15 @@ func (i *Test) prepareShutdownHandlers() { i.ingestNode.Close() } if !i.config.SkipCoreContainerCreation { - if !i.config.SkipCoreContainerDeletion { - i.t.Log("Removing core docker containers...") - i.runComposeCommand("rm", "-fvs", "core") - i.runComposeCommand("rm", "-fvs", "core-postgres") - } else { - i.t.Log("Skip core docker container removal for debugging...") + if i.config.LogContainers { + i.runComposeCommand(nil, "logs", "core") } - if i.config.EnableSorobanRPC { - i.runComposeCommand("logs", "soroban-rpc") - i.runComposeCommand("rm", "-fvs", "soroban-rpc") + i.removeContainers("core") + if i.config.EnableStellarRPC { + if i.config.LogContainers { + i.runComposeCommand(nil, "logs", "stellar-rpc") + } + i.removeContainers("stellar-rpc") } } }, @@ -587,45 +649,14 @@ func (i *Test) setupHorizonClient(webArgs map[string]string) { } } -// CreateCaptiveCoreConfig will create a temporary TOML config with the -// specified contents as well as a temporary storage directory. You should -// `defer` the returned function to clean these up when you're done. -func CreateCaptiveCoreConfig(contents string) (string, string, func()) { - tomlFile, err := ioutil.TempFile("", "captive-core-test-*.toml") - if err != nil { - panic(err) - } - defer tomlFile.Close() - - _, err = tomlFile.WriteString(contents) - if err != nil { - panic(err) - } - - storagePath, err := os.MkdirTemp("", "captive-core-test-*-storage") - if err != nil { - panic(err) - } - - filename := tomlFile.Name() - return filename, storagePath, func() { - os.Remove(filename) - os.RemoveAll(storagePath) - } -} - func (i *Test) CreateCaptiveCoreConfig() (*ledgerbackend.CaptiveCoreConfig, error) { - confName, storagePath, cleanupFn := CreateCaptiveCoreConfig(SimpleCaptiveCoreToml) - i.t.Cleanup(cleanupFn) - i.t.Logf("Creating Captive Core config files, ConfName: %v, storagePath: %v", confName, storagePath) - captiveCoreConfig := ledgerbackend.CaptiveCoreConfig{ - BinaryPath: i.coreConfig.binaryPath, + BinaryPath: i.CoreBinaryPath(), HistoryArchiveURLs: []string{HistoryArchiveUrl}, NetworkPassphrase: StandaloneNetworkPassphrase, CheckpointFrequency: CheckpointFrequency, // This is required for accelerated archive creation for integration test UseDB: true, - StoragePath: storagePath, + StoragePath: i.CurrentTest().TempDir(), } tomlParams := ledgerbackend.CaptiveCoreTomlParams{ @@ -634,7 +665,7 @@ func (i *Test) CreateCaptiveCoreConfig() (*ledgerbackend.CaptiveCoreConfig, erro UseDB: true, } - toml, err := ledgerbackend.NewCaptiveCoreTomlFromData([]byte(SimpleCaptiveCoreToml), tomlParams) + toml, err := ledgerbackend.NewCaptiveCoreTomlFromFile(i.coreConfig.configPath, tomlParams) if err != nil { return nil, err } @@ -689,35 +720,35 @@ func (i *Test) waitForCore() { i.t.Fatalf("Core could not sync after %v + %v", maxWaitForCoreStartup, maxWaitForCoreUpgrade) } -const sorobanRPCInitTime = 20 * time.Second -const sorobanRPCHealthCheckInterval = time.Second +const stellarRPCInitTime = 60 * 6 * time.Second +const stellarRPCHealthCheckInterval = time.Second -// Wait for SorobanRPC to be up -func (i *Test) waitForSorobanRPC() { - i.t.Log("Waiting for Soroban RPC to be up...") +// Wait for stellar rpc to be up +func (i *Test) waitForStellarRPC() { + i.t.Log("Waiting for Stellar RPC to be up...") start := time.Now() - for time.Since(start) < sorobanRPCInitTime { - ctx, cancel := context.WithTimeout(context.Background(), sorobanRPCHealthCheckInterval) + for time.Since(start) < stellarRPCInitTime { + ctx, cancel := context.WithTimeout(context.Background(), stellarRPCHealthCheckInterval) // TODO: soroban-tools should be exporting a proper Go client - ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(SorobanRPCPort), nil) - sorobanRPCClient := jrpc2.NewClient(ch, nil) + ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(StellarRPCPort), nil) + stellarRPCClient := jrpc2.NewClient(ch, nil) callTime := time.Now() - _, err := sorobanRPCClient.Call(ctx, "getHealth", nil) + _, err := stellarRPCClient.Call(ctx, "getHealth", nil) cancel() if err != nil { - i.t.Logf("SorobanRPC is unhealthy: %v", err) + i.t.Logf("stellar rpc is unhealthy: %v", err) // sleep up to a second between consecutive calls. - if durationSince := time.Since(callTime); durationSince < sorobanRPCHealthCheckInterval { - time.Sleep(sorobanRPCHealthCheckInterval - durationSince) + if durationSince := time.Since(callTime); durationSince < stellarRPCHealthCheckInterval { + time.Sleep(stellarRPCHealthCheckInterval - durationSince) } continue } - i.t.Log("SorobanRPC is up.") + i.t.Log("stellar rpc is up.") return } - i.t.Fatalf("SorobanRPC unhealthy after %v", time.Since(start)) + i.t.Fatalf("stellar rpc unhealthy after %v", time.Since(start)) } type RPCSimulateHostFunctionResult struct { @@ -765,14 +796,14 @@ func (i *Test) PreflightHostFunctions( func (i *Test) simulateTransaction( sourceAccount txnbuild.Account, op txnbuild.Operation, ) (RPCSimulateTxResponse, xdr.SorobanTransactionData) { - // Before preflighting, make sure soroban-rpc is in sync with Horizon + // Before preflighting, make sure stellar-rpc is in sync with Horizon root, err := i.horizonClient.Root() assert.NoError(i.t, err) - i.syncWithSorobanRPC(uint32(root.HorizonSequence)) + i.syncWithStellarRPC(uint32(root.HorizonSequence)) // TODO: soroban-tools should be exporting a proper Go client - ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(SorobanRPCPort), nil) - sorobanRPCClient := jrpc2.NewClient(ch, nil) + ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(StellarRPCPort), nil) + stellarRPCClient := jrpc2.NewClient(ch, nil) txParams := GetBaseTransactionParamsWithFee(sourceAccount, txnbuild.MinBaseFee, op) txParams.IncrementSequenceNum = false tx, err := txnbuild.NewTransaction(txParams) @@ -781,7 +812,7 @@ func (i *Test) simulateTransaction( assert.NoError(i.t, err) result := RPCSimulateTxResponse{} fmt.Printf("Preflight TX:\n\n%v \n\n", base64) - err = sorobanRPCClient.CallResult(context.Background(), "simulateTransaction", struct { + err = stellarRPCClient.CallResult(context.Background(), "simulateTransaction", struct { Transaction string `json:"transaction"` }{base64}, &result) assert.NoError(i.t, err) @@ -793,25 +824,25 @@ func (i *Test) simulateTransaction( return result, transactionData } -func (i *Test) syncWithSorobanRPC(ledgerToWaitFor uint32) { +func (i *Test) syncWithStellarRPC(ledgerToWaitFor uint32) { for j := 0; j < 20; j++ { result := struct { Sequence uint32 `json:"sequence"` }{} - ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(SorobanRPCPort), nil) - sorobanRPCClient := jrpc2.NewClient(ch, nil) - err := sorobanRPCClient.CallResult(context.Background(), "getLatestLedger", nil, &result) + ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(StellarRPCPort), nil) + stellarRPCClient := jrpc2.NewClient(ch, nil) + err := stellarRPCClient.CallResult(context.Background(), "getLatestLedger", nil, &result) assert.NoError(i.t, err) if result.Sequence >= ledgerToWaitFor { return } time.Sleep(500 * time.Millisecond) } - i.t.Fatal("Time out waiting for soroban-rpc to sync") + i.t.Fatal("Time out waiting for stellar-rpc to sync") } func (i *Test) WaitUntilLedgerEntryTTL(ledgerKey xdr.LedgerKey) { - ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(SorobanRPCPort), nil) + ch := jhttp.NewChannel("http://localhost:"+strconv.Itoa(StellarRPCPort), nil) client := jrpc2.NewClient(ch, nil) keyB64, err := xdr.MarshalBase64(ledgerKey) @@ -842,7 +873,7 @@ func (i *Test) WaitUntilLedgerEntryTTL(ledgerKey xdr.LedgerKey) { } i.t.Log("waiting for ledger entry to ttl at ledger", liveUntilLedgerSeq) } else { - i.t.Log("waiting for soroban-rpc to ingest the ledger entries") + i.t.Log("waiting for stellar-rpc to ingest the ledger entries") } time.Sleep(time.Second) } @@ -998,6 +1029,19 @@ func (i *Test) CoreClient() *stellarcore.Client { return i.coreClient } +func (i *Test) CoreBinaryPath() string { + if i.coreConfig.binaryPath != "" { + return i.coreConfig.binaryPath + } + corePath, err := exec.LookPath("stellar-core") + require.NoError(i.t, err) + return corePath +} + +func (i *Test) CaptiveCoreStoragePath() string { + return i.coreConfig.storagePath +} + // Client returns horizon.Client connected to started Horizon instance. func (i *Test) Client() *sdk.Client { return i.horizonClient