feat(nns): Add pagination to list_neurons API

rs/nervous_system/integration_tests/src/pocket_ic_helpers.rs

@@ -1,12 +1,13 @@
 use candid::{Decode, Encode, Nat, Principal};
 use canister_test::Wasm;
-use futures::stream;
-use futures::StreamExt;
+use futures::{stream, StreamExt};
 use ic_base_types::{CanisterId, PrincipalId, SubnetId};
 use ic_ledger_core::Tokens;
-use ic_nervous_system_agent::pocketic_impl::{PocketIcAgent, PocketIcCallError};
-use ic_nervous_system_agent::sns::Sns;
-use ic_nervous_system_agent::CallCanisters;
+use ic_nervous_system_agent::{
+    pocketic_impl::{PocketIcAgent, PocketIcCallError},
+    sns::Sns,
+    CallCanisters,
+};
 use ic_nervous_system_common::{E8, ONE_DAY_SECONDS};
 use ic_nervous_system_common_test_keys::{TEST_NEURON_1_ID, TEST_NEURON_1_OWNER_PRINCIPAL};
 use ic_nns_common::pb::v1::{NeuronId, ProposalId};
@@ -60,17 +61,15 @@ use icrc_ledger_types::icrc1::{
     account::Account,
     transfer::{TransferArg, TransferError},
 };
-use itertools::EitherOrBoth;
-use itertools::Itertools;
+use itertools::{EitherOrBoth, Itertools};
 use maplit::btreemap;
 use pocket_ic::{
     management_canister::CanisterSettings, nonblocking::PocketIc, ErrorCode, PocketIcBuilder,
     RejectResponse,
 };
 use prost::Message;
 use rust_decimal::prelude::ToPrimitive;
-use std::ops::Range;
-use std::{collections::BTreeMap, fmt::Write, time::Duration};
+use std::{collections::BTreeMap, fmt::Write, ops::Range, time::Duration};
 
 pub const STARTING_CYCLES_PER_CANISTER: u128 = 2_000_000_000_000_000;
 
@@ -833,6 +832,8 @@ pub mod nns {
                         include_neurons_readable_by_caller: true,
                         include_empty_neurons_readable_by_caller: None,
                         include_public_neurons_in_full_neurons: None,
+                        page_number: None,
+                        page_size: None
                     })
                     .unwrap(),
                 )
@@ -1366,8 +1367,9 @@ pub mod sns {
         use assert_matches::assert_matches;
         use ic_crypto_sha2::Sha256;
         use ic_nervous_system_agent::sns::governance::{GovernanceCanister, SubmitProposalError};
-        use ic_sns_governance::governance::UPGRADE_STEPS_INTERVAL_REFRESH_BACKOFF_SECONDS;
-        use ic_sns_governance::pb::v1::get_neuron_response;
+        use ic_sns_governance::{
+            governance::UPGRADE_STEPS_INTERVAL_REFRESH_BACKOFF_SECONDS, pb::v1::get_neuron_response,
+        };
         use pocket_ic::ErrorCode;
         use sns_pb::UpgradeSnsControlledCanister;
 

rs/nns/governance/api/src/ic_nns_governance.pb.v1.rs

@@ -3320,6 +3320,20 @@ pub struct ListProposalInfoResponse {
 /// of neurons listed in `neuron_ids` and, if `caller_neurons` is true,
 /// the list of neuron IDs of neurons for which the caller is the
 /// controller or one of the hot keys.
+///
+/// Paging is available if the result set is larger than `MAX_LIST_NEURONS_RESULTS`,
+/// which is currently 500 neurons.  If you are unsure of the number of results in a set,
+/// you can use the `total_pages_available` field in the response to determine how many
+/// additional pages need to be queried.  It will be based on your `page_size` parameter.  
+/// When paging through results, it is good to keep in mind that newly inserted neurons
+/// could be missed if they are inserted between calls to pages, and this could result in missing
+/// a neuron in the combined responses.
+///
+/// If a user provides neuron_ids that do not exist in the request, there is no guarantee that
+/// each page will contain the exactly the page size, even if it is not the final request.  This is
+/// because neurons are retrieved by their neuron_id, and no additional checks are made on the
+/// validity of the neuron_ids provided by the user before deciding which sets of neuron_ids
+/// will be returned in the current page.
 #[derive(candid::CandidType, candid::Deserialize, serde::Serialize, comparable::Comparable)]
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -3350,6 +3364,15 @@ pub struct ListNeurons {
     /// existing (unmigrated) callers.
     #[prost(bool, optional, tag = "4")]
     pub include_public_neurons_in_full_neurons: Option<bool>,
+    /// If this is set, we return the batch of neurons at a given page, using the `page_size` to
+    /// determine how many neurons are returned in each page.
+    #[prost(uint64, optional, tag = "5")]
+    pub page_number: Option<u64>,
+    /// If this is set, we use the page limit provided to determine how large pages will be.
+    /// This cannot be greater than MAX_LIST_NEURONS_RESULTS, which is set to 500.
+    /// If not set, this defaults to MAX_LIST_NEURONS_RESULTS.
+    #[prost(uint64, optional, tag = "6")]
+    pub page_size: Option<u64>,
 }
 /// A response to a `ListNeurons` request.
 ///
@@ -3368,6 +3391,10 @@ pub struct ListNeuronsResponse {
     /// `ManageNeuron` topic).
     #[prost(message, repeated, tag = "2")]
     pub full_neurons: Vec<Neuron>,
+    /// This is returned to tell the caller how many pages of results are available to query.
+    /// If there are fewer than the page_size neurons, this will equal 1.
+    #[prost(uint64, optional, tag = "3")]
+    pub total_pages_available: Option<u64>,
 }
 /// A response to "ListKnownNeurons"
 #[derive(candid::CandidType, candid::Deserialize, serde::Serialize, comparable::Comparable)]

rs/nns/governance/canbench/canbench_results.yml

@@ -1,55 +1,55 @@
 benches:
   add_neuron_active_maximum:
     total:
-      instructions: 42752805
+      instructions: 42749561
       heap_increase: 1
       stable_memory_increase: 0
     scopes: {}
   add_neuron_active_typical:
     total:
-      instructions: 2170667
+      instructions: 2170531
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   add_neuron_inactive_maximum:
     total:
-      instructions: 112624643
+      instructions: 112621399
       heap_increase: 1
       stable_memory_increase: 0
     scopes: {}
   add_neuron_inactive_typical:
     total:
-      instructions: 8497304
+      instructions: 8497168
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   cascading_vote_all_heap:
     total:
-      instructions: 35002536
+      instructions: 34977262
       heap_increase: 0
       stable_memory_increase: 128
     scopes: {}
   cascading_vote_heap_neurons_stable_index:
     total:
-      instructions: 61137575
+      instructions: 61112301
       heap_increase: 0
       stable_memory_increase: 128
     scopes: {}
   cascading_vote_stable_everything:
     total:
-      instructions: 188621982
+      instructions: 188615092
       heap_increase: 0
       stable_memory_increase: 128
     scopes: {}
   cascading_vote_stable_neurons_with_heap_index:
     total:
-      instructions: 162353547
+      instructions: 162346657
       heap_increase: 0
       stable_memory_increase: 128
     scopes: {}
   centralized_following_all_stable:
     total:
-      instructions: 78268135
+      instructions: 78266673
       heap_increase: 0
       stable_memory_increase: 128
     scopes: {}
@@ -61,7 +61,7 @@ benches:
     scopes: {}
   draw_maturity_from_neurons_fund_heap:
     total:
-      instructions: 7598030
+      instructions: 7584430
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
@@ -85,7 +85,7 @@ benches:
     scopes: {}
   list_neurons_heap:
     total:
-      instructions: 4763239
+      instructions: 4802543
       heap_increase: 9
       stable_memory_increase: 0
     scopes: {}
@@ -103,13 +103,13 @@ benches:
     scopes: {}
   list_neurons_stable:
     total:
-      instructions: 113374022
+      instructions: 113411672
       heap_increase: 5
       stable_memory_increase: 0
     scopes: {}
   list_proposals:
     total:
-      instructions: 168095717
+      instructions: 168095740
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
@@ -127,13 +127,13 @@ benches:
     scopes: {}
   neuron_data_validation_heap:
     total:
-      instructions: 406864991
+      instructions: 406848584
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   neuron_data_validation_stable:
     total:
-      instructions: 362661286
+      instructions: 362648972
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
@@ -151,13 +151,13 @@ benches:
     scopes: {}
   range_neurons_performance:
     total:
-      instructions: 56448740
+      instructions: 56447340
       heap_increase: 0
       stable_memory_increase: 0
     scopes: {}
   single_vote_all_stable:
     total:
-      instructions: 2805838
+      instructions: 2803168
       heap_increase: 0
       stable_memory_increase: 128
     scopes: {}

rs/nns/governance/canister/canister.rs

@@ -888,7 +888,10 @@ fn get_latest_reward_event() -> RewardEvent {
 #[query]
 fn get_neuron_ids() -> Vec<NeuronId> {
     debug_log("get_neuron_ids");
-    let votable = governance().get_neuron_ids_by_principal(&caller());
+    let votable = governance()
+        .get_neuron_ids_by_principal(&caller())
+        .into_iter()
+        .collect();
 
     governance()
         .get_managed_neuron_ids_for(votable)

rs/nns/governance/canister/governance.did

@@ -439,11 +439,14 @@ type ListNeurons = record {
   neuron_ids : vec nat64;
   include_empty_neurons_readable_by_caller : opt bool;
   include_neurons_readable_by_caller : bool;
+  page_number: opt nat64;
+  page_size: opt nat64;
 };
 
 type ListNeuronsResponse = record {
   neuron_infos : vec record { nat64; NeuronInfo };
   full_neurons : vec Neuron;
+  total_pages_available: opt nat64;
 };
 
 type ListNodeProviderRewardsRequest = record {

rs/nns/governance/canister/governance_test.did

@@ -438,11 +438,14 @@ type ListNeurons = record {
   neuron_ids : vec nat64;
   include_empty_neurons_readable_by_caller : opt bool;
   include_neurons_readable_by_caller : bool;
+  page_number: opt nat64;
+  page_size: opt nat64;
 };
 
 type ListNeuronsResponse = record {
   neuron_infos : vec record { nat64; NeuronInfo };
   full_neurons : vec Neuron;
+  total_pages_available: opt nat64;
 };
 
 type ListNodeProviderRewardsRequest = record {

rs/nns/governance/proto/ic_nns_governance/pb/v1/governance.proto

@@ -2635,6 +2635,20 @@ message ListProposalInfoResponse {
 // of neurons listed in `neuron_ids` and, if `caller_neurons` is true,
 // the list of neuron IDs of neurons for which the caller is the
 // controller or one of the hot keys.
+//
+// Paging is available if the result set is larger than `MAX_LIST_NEURONS_RESULTS`,
+// which is currently 500 neurons.  If you are unsure of the number of results in a set,
+// you can use the `total_pages_available` field in the response to determine how many
+// additional pages need to be queried.  It will be based on your `page_size` parameter.
+// When paging through results, it is good to keep in mind that newly inserted neurons
+// could be missed if they are inserted between calls to pages, and this could result in missing
+// a neuron in the combined responses.
+//
+// If a user provides neuron_ids that do not exist in the request, there is no guarantee that
+// each page will contain the exactly the page size, even if it is not the final request.  This is
+// because neurons are retrieved by their neuron_id, and no additional checks are made on the
+// validity of the neuron_ids provided by the user before deciding which sets of neuron_ids
+// will be returned in the current page.
 message ListNeurons {
   option (ic_base_types.pb.v1.tui_signed_message) = true;
   // The neurons to get information about. The "requested list"
@@ -2659,6 +2673,13 @@ message ListNeurons {
   // since this feature was added later, it is opt in to avoid confusing
   // existing (unmigrated) callers.
   optional bool include_public_neurons_in_full_neurons = 4;
+  // If this is set, we return the batch of neurons at a given page, using the `page_size` to
+  // determine how many neurons are returned in each page.
+  optional uint64 page_number = 5;
+  // If this is set, we use the page limit provided to determine how large pages will be.
+  // This cannot be greater than MAX_LIST_NEURONS_RESULTS, which is set to 500.
+  // If not set, this defaults to MAX_LIST_NEURONS_RESULTS.
+  optional uint64 page_size = 6;
 }
 
 // A response to a `ListNeurons` request.
@@ -2673,6 +2694,9 @@ message ListNeuronsResponse {
   // hot key, or controller or hot key of some followee on the
   // `ManageNeuron` topic).
   repeated Neuron full_neurons = 2;
+  // This is returned to tell the caller how many pages of results are available to query.
+  // If there are fewer than the page_size neurons, this will equal 1.
+  optional fixed64 total_pages_available = 3;
 }
 
 // A response to "ListKnownNeurons"

rs/nns/governance/src/gen/ic_nns_governance.pb.v1.rs

@@ -3932,6 +3932,20 @@ pub struct ListProposalInfoResponse {
 /// of neurons listed in `neuron_ids` and, if `caller_neurons` is true,
 /// the list of neuron IDs of neurons for which the caller is the
 /// controller or one of the hot keys.
+///
+/// Paging is available if the result set is larger than `MAX_LIST_NEURONS_RESULTS`,
+/// which is currently 500 neurons.  If you are unsure of the number of results in a set,
+/// you can use the `total_pages_available` field in the response to determine how many
+/// additional pages need to be queried.  It will be based on your `page_size` parameter.
+/// When paging through results, it is good to keep in mind that newly inserted neurons
+/// could be missed if they are inserted between calls to pages, and this could result in missing
+/// a neuron in the combined responses.
+///
+/// If a user provides neuron_ids that do not exist in the request, there is no guarantee that
+/// each page will contain the exactly the page size, even if it is not the final request.  This is
+/// because neurons are retrieved by their neuron_id, and no additional checks are made on the
+/// validity of the neuron_ids provided by the user before deciding which sets of neuron_ids
+/// will be returned in the current page.
 #[derive(
     candid::CandidType,
     candid::Deserialize,
@@ -3968,6 +3982,15 @@ pub struct ListNeurons {
     /// existing (unmigrated) callers.
     #[prost(bool, optional, tag = "4")]
     pub include_public_neurons_in_full_neurons: ::core::option::Option<bool>,
+    /// If this is set, we return the batch of neurons at a given page, using the `page_size` to
+    /// determine how many neurons are returned in each page.
+    #[prost(uint64, optional, tag = "5")]
+    pub page_number: ::core::option::Option<u64>,
+    /// If this is set, we use the page limit provided to determine how large pages will be.
+    /// This cannot be greater than MAX_LIST_NEURONS_RESULTS, which is set to 500.
+    /// If not set, this defaults to MAX_LIST_NEURONS_RESULTS.
+    #[prost(uint64, optional, tag = "6")]
+    pub page_size: ::core::option::Option<u64>,
 }
 /// A response to a `ListNeurons` request.
 ///
@@ -3992,6 +4015,10 @@ pub struct ListNeuronsResponse {
     /// `ManageNeuron` topic).
     #[prost(message, repeated, tag = "2")]
     pub full_neurons: ::prost::alloc::vec::Vec<Neuron>,
+    /// This is returned to tell the caller how many pages of results are available to query.
+    /// If there are fewer than the page_size neurons, this will equal 1.
+    #[prost(fixed64, optional, tag = "3")]
+    pub total_pages_available: ::core::option::Option<u64>,
 }
 /// A response to "ListKnownNeurons"
 #[derive(