From dd5ce4fd7d956d85ec82c08f83d40b8eeffea3a8 Mon Sep 17 00:00:00 2001 From: mraszyk <31483726+mraszyk@users.noreply.github.com> Date: Mon, 28 Oct 2024 21:13:20 +0100 Subject: [PATCH] fix: use https in PocketIC specification compliance tests (#2246) This PR makes the PocketIC specification compliance tests (the test target `//rs/pocket_ic_server:spec_test`) use https for canister http outcalls. This way, those (pre-master) tests match the specification compliance system tests. --------- Co-authored-by: Daniel Sharifi --- hs/spec_compliance/src/IC/Test/Spec/HTTP.hs | 34 ++++-------- rs/pocket_ic_server/tests/spec_test.rs | 61 ++++++++++++++++++--- rs/tests/httpbin-rs/src/main.rs | 14 +++-- 3 files changed, 74 insertions(+), 35 deletions(-) diff --git a/hs/spec_compliance/src/IC/Test/Spec/HTTP.hs b/hs/spec_compliance/src/IC/Test/Spec/HTTP.hs index fd30328088b..60e123ad5d2 100644 --- a/hs/spec_compliance/src/IC/Test/Spec/HTTP.hs +++ b/hs/spec_compliance/src/IC/Test/Spec/HTTP.hs @@ -195,11 +195,10 @@ canister_http_calls sub httpbin_proto = date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 11 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 158 + let header_size = 143 resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("ascii/" ++ s) (Just $ fromIntegral $ length s + header_size) Nothing cid (resp .! #status) @?= 200 (resp .! #body) @?= BLU.fromString s @@ -210,22 +209,20 @@ canister_http_calls sub httpbin_proto = date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 11 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 158 + let header_size = 143 ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("ascii/" ++ s) (Just $ fromIntegral $ length s + header_size - 1) Nothing cid >>= isReject [1], simpleTestCase "small maximum possible response size (only headers)" ecid $ \cid -> do {- Response headers (size: 157) date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 0 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 157 + let header_size = 142 resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/0") (Just header_size) Nothing cid (resp .! #status) @?= 200 (resp .! #body) @?= BS.empty @@ -235,11 +232,10 @@ canister_http_calls sub httpbin_proto = date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 0 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 157 + let header_size = 142 ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/0") (Just $ header_size - 1) Nothing cid >>= isReject [1], -- "The upper limit on the maximal size for the response is 2MB (2,000,000B) and this value also applies if no maximal size value is specified." @@ -248,11 +244,10 @@ canister_http_calls sub httpbin_proto = date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 1999837 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 163 + let header_size = 148 cid <- install ecid (onTransform (callback (replyData (bytes (Candid.encode dummyResponse))))) resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/" ++ show (max_response_bytes_limit - header_size)) Nothing (Just ("transform", "")) cid (resp .! #status) @?= 202 @@ -263,11 +258,10 @@ canister_http_calls sub httpbin_proto = date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 1999838 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 163 + let header_size = 148 cid <- install ecid (onTransform (callback (replyData (bytes (Candid.encode dummyResponse))))) ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/" ++ show (max_response_bytes_limit - header_size + 1)) Nothing (Just ("transform", "")) cid >>= isReject [1], -- "The URL must be valid according to RFC-3986 and its length must not exceed 8192." @@ -326,11 +320,10 @@ canister_http_calls sub httpbin_proto = date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 1999837 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 163 + let header_size = 148 let size = maximumSizeResponseBodySize let new_pages = int $ size `div` (64 * 1024) + 1 let max_size = int $ size @@ -343,11 +336,10 @@ canister_http_calls sub httpbin_proto = date: Jan 1 1970 00:00:00 GMT content-type: application/octet-stream content-length: 1999837 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let header_size = 163 + let header_size = 148 let size = maximumSizeResponseBodySize + 1 let new_pages = int $ size `div` (64 * 1024) + 1 let max_size = int $ size @@ -375,30 +367,28 @@ canister_http_calls sub httpbin_proto = let hs = [(T.pack ("name" ++ show i), T.pack ("value" ++ show i)) | i <- [0 .. http_headers_max_number]] ic_http_post_request' (\fee -> ic00viaWithCyclesRefund fee cid fee) sub httpbin_proto "anything" Nothing (Just b) (vec_header_from_list_text hs) Nothing cid >>= isReject [4], simpleTestCase "maximum number of response headers" ecid $ \cid -> do - {- These 6 response headers are always included: + {- These 5 response headers are always included: date: Jan 1 1970 00:00:00 GMT content-type: text/plain; charset=utf-8 content-length: 0 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let n = http_headers_max_number - 6 + let n = http_headers_max_number - 5 let hs = [(T.pack ("name" ++ show i), T.pack ("value" ++ show i)) | i <- [0 .. n - 1]] resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("many_response_headers/" ++ show n) Nothing Nothing cid (resp .! #status) @?= 200 assertBool "Response HTTP headers have not been received properly." $ list_subset (map_to_lower hs) (map_to_lower $ http_response_headers resp) check_http_response resp, simpleTestCase "maximum number of response headers exceeded" ecid $ \cid -> do - {- These 6 response headers are always included: + {- These 5 response headers are always included: date: Jan 1 1970 00:00:00 GMT content-type: text/plain; charset=utf-8 content-length: 0 - connection: close access-control-allow-origin: * access-control-allow-credentials: true -} - let n = http_headers_max_number - 6 + 1 + let n = http_headers_max_number - 5 + 1 ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("many_response_headers/" ++ show n) Nothing Nothing cid >>= isReject [1], -- "The following additional limits apply to HTTP requests and HTTP responses from the remote sever: the number of bytes representing a header name must not exceed 8KiB." diff --git a/rs/pocket_ic_server/tests/spec_test.rs b/rs/pocket_ic_server/tests/spec_test.rs index 7a744209f65..4d3213cb078 100644 --- a/rs/pocket_ic_server/tests/spec_test.rs +++ b/rs/pocket_ic_server/tests/spec_test.rs @@ -6,13 +6,13 @@ use ic_registry_routing_table::{canister_id_into_u64, CanisterIdRange}; use ic_registry_subnet_type::SubnetType; use pocket_ic::common::rest::DtsFlag; use pocket_ic::PocketIcBuilder; +use rcgen::{CertificateParams, KeyPair}; use spec_compliance::run_ic_ref_test; +use std::io::Write; use std::process::{Command, Stdio}; use std::time::Duration; use tempfile::NamedTempFile; -const LOCALHOST: &str = "127.0.0.1"; - const EXCLUDED: &[&str] = &[ // we do not enforce https in PocketIC "$0 ~ /url must start with https:/", @@ -53,14 +53,44 @@ fn subnet_config( ) } -fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included_tests: Vec<&str>) { - // start httpbin webserver to test canister HTTP outcalls +// `httpbin_https` can only be set to `true` in a single test +fn setup_and_run_ic_ref_test( + test_nns: bool, + httpbin_https: bool, + excluded_tests: Vec<&str>, + included_tests: Vec<&str>, +) { + // start httpbin webserver to test canister http outcalls let httpbin_path = std::env::var_os("HTTPBIN_BIN").expect("Missing httpbin binary path"); let mut cmd = Command::new(httpbin_path); let port_file = NamedTempFile::new().unwrap(); let port_file_path = port_file.path().to_path_buf(); cmd.arg("--port-file") .arg(port_file_path.as_os_str().to_str().unwrap()); + + if httpbin_https { + // generate root TLS certificate + let root_key_pair = KeyPair::generate().unwrap(); + let root_cert = CertificateParams::new(vec!["localhost".to_string()]) + .unwrap() + .self_signed(&root_key_pair) + .unwrap(); + let (mut cert_file, cert_path) = NamedTempFile::new().unwrap().keep().unwrap(); + cert_file.write_all(root_cert.pem().as_bytes()).unwrap(); + let (mut key_file, key_path) = NamedTempFile::new().unwrap().keep().unwrap(); + key_file + .write_all(root_key_pair.serialize_pem().as_bytes()) + .unwrap(); + + // set `SSL_CERT_FILE` so that the canister http outcalls adapter accepts the self-signed certificate + // (this affects all tests and thus `httbin_https` should only be set to `true` in a single test) + std::env::set_var("SSL_CERT_FILE", cert_path.clone()); + std::env::remove_var("NIX_SSL_CERT_FILE"); + + cmd.arg("--cert-file").arg(cert_path); + cmd.arg("--key-file").arg(key_path); + } + cmd.stdout(Stdio::inherit()) .stderr(Stdio::inherit()) .spawn() @@ -73,7 +103,7 @@ fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included .trim_end() .parse() .expect("Failed to parse port to number"); - break format!("{}:{}", LOCALHOST, port); + break format!("localhost:{}", port); } std::thread::sleep(Duration::from_millis(20)); }; @@ -151,8 +181,13 @@ fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included nns_subnet_config }; + let httpbin_proto = if httpbin_https { + Some("https://".to_string()) + } else { + Some("http://".to_string()) + }; run_ic_ref_test( - Some("http://".to_string()), + httpbin_proto, Some(httpbin_url), ic_ref_test_path.into_os_string().into_string().unwrap(), ic_test_data_path, @@ -167,10 +202,20 @@ fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included #[test] fn ic_ref_test_nns() { - setup_and_run_ic_ref_test(true, EXCLUDED.to_vec(), vec![]) + setup_and_run_ic_ref_test(true, false, EXCLUDED.to_vec(), vec![]) } #[test] fn ic_ref_test_app() { - setup_and_run_ic_ref_test(false, EXCLUDED.to_vec(), vec![]) + setup_and_run_ic_ref_test(false, true, EXCLUDED.to_vec(), vec![]) +} + +#[test] +fn ic_ref_test_canister_http() { + setup_and_run_ic_ref_test( + false, + false, + EXCLUDED.to_vec(), + vec!["$0 ~ /canister http/"], + ) } diff --git a/rs/tests/httpbin-rs/src/main.rs b/rs/tests/httpbin-rs/src/main.rs index be03a04b37a..b10737b7a48 100644 --- a/rs/tests/httpbin-rs/src/main.rs +++ b/rs/tests/httpbin-rs/src/main.rs @@ -32,13 +32,15 @@ use tokio::{ use tokio_rustls::TlsAcceptor; use tower::Service; -const DETERMINISTIC_HEADERS: [(&str, &str); 4] = [ - ("Access-Control-Allow-Origin", "*"), - ("Access-Control-Allow-Credentials", "true"), - ("Connection", "close"), - ("Date", "Jan 1 1970 00:00:00 GMT"), +const DETERMINISTIC_HEADERS: [(&str, &str); 3] = [ + ("access-control-allow-origin", "*"), + ("access-control-allow-credentials", "true"), + ("date", "Jan 1 1970 00:00:00 GMT"), ]; +/// Set a very large limit for the headers to accept. +const MAX_HEADER_LIST_SIZE: u32 = 1024 * 1024; // 1MiB + /// Returns a normal HTML response async fn root_handler() -> Html<&'static str> { Html( @@ -310,6 +312,8 @@ async fn main() -> Result<(), Box> { } }; if let Err(err) = Builder::new(TokioExecutor::new()) + .http2() + .max_header_list_size(MAX_HEADER_LIST_SIZE) .serve_connection(TokioIo::new(tls_stream), hyper_service) .await {