fix: use https in PocketIC specification compliance tests (#2246)

This PR makes the PocketIC specification compliance tests (the test
target `//rs/pocket_ic_server:spec_test`) use https for canister http
outcalls. This way, those (pre-master) tests match the specification
compliance system tests.

---------

Co-authored-by: Daniel Sharifi <[email protected]>

hs/spec_compliance/src/IC/Test/Spec/HTTP.hs

@@ -195,11 +195,10 @@ canister_http_calls sub httpbin_proto =
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 11
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 158
+              let header_size = 143
               resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("ascii/" ++ s) (Just $ fromIntegral $ length s + header_size) Nothing cid
               (resp .! #status) @?= 200
               (resp .! #body) @?= BLU.fromString s
@@ -210,22 +209,20 @@ canister_http_calls sub httpbin_proto =
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 11
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 158
+              let header_size = 143
               ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("ascii/" ++ s) (Just $ fromIntegral $ length s + header_size - 1) Nothing cid >>= isReject [1],
             simpleTestCase "small maximum possible response size (only headers)" ecid $ \cid -> do
               {- Response headers (size: 157)
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 0
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 157
+              let header_size = 142
               resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/0") (Just header_size) Nothing cid
               (resp .! #status) @?= 200
               (resp .! #body) @?= BS.empty
@@ -235,11 +232,10 @@ canister_http_calls sub httpbin_proto =
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 0
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 157
+              let header_size = 142
               ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/0") (Just $ header_size - 1) Nothing cid >>= isReject [1],
             -- "The upper limit on the maximal size for the response is 2MB (2,000,000B) and this value also applies if no maximal size value is specified."
 
@@ -248,11 +244,10 @@ canister_http_calls sub httpbin_proto =
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 1999837
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 163
+              let header_size = 148
               cid <- install ecid (onTransform (callback (replyData (bytes (Candid.encode dummyResponse)))))
               resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/" ++ show (max_response_bytes_limit - header_size)) Nothing (Just ("transform", "")) cid
               (resp .! #status) @?= 202
@@ -263,11 +258,10 @@ canister_http_calls sub httpbin_proto =
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 1999838
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 163
+              let header_size = 148
               cid <- install ecid (onTransform (callback (replyData (bytes (Candid.encode dummyResponse)))))
               ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("equal_bytes/" ++ show (max_response_bytes_limit - header_size + 1)) Nothing (Just ("transform", "")) cid >>= isReject [1],
             -- "The URL must be valid according to RFC-3986 and its length must not exceed 8192."
@@ -326,11 +320,10 @@ canister_http_calls sub httpbin_proto =
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 1999837
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 163
+              let header_size = 148
               let size = maximumSizeResponseBodySize
               let new_pages = int $ size `div` (64 * 1024) + 1
               let max_size = int $ size
@@ -343,11 +336,10 @@ canister_http_calls sub httpbin_proto =
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: application/octet-stream
                   content-length: 1999837
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let header_size = 163
+              let header_size = 148
               let size = maximumSizeResponseBodySize + 1
               let new_pages = int $ size `div` (64 * 1024) + 1
               let max_size = int $ size
@@ -375,30 +367,28 @@ canister_http_calls sub httpbin_proto =
               let hs = [(T.pack ("name" ++ show i), T.pack ("value" ++ show i)) | i <- [0 .. http_headers_max_number]]
               ic_http_post_request' (\fee -> ic00viaWithCyclesRefund fee cid fee) sub httpbin_proto "anything" Nothing (Just b) (vec_header_from_list_text hs) Nothing cid >>= isReject [4],
             simpleTestCase "maximum number of response headers" ecid $ \cid -> do
-              {- These 6 response headers are always included:
+              {- These 5 response headers are always included:
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: text/plain; charset=utf-8
                   content-length: 0
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let n = http_headers_max_number - 6
+              let n = http_headers_max_number - 5
               let hs = [(T.pack ("name" ++ show i), T.pack ("value" ++ show i)) | i <- [0 .. n - 1]]
               resp <- ic_http_get_request (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("many_response_headers/" ++ show n) Nothing Nothing cid
               (resp .! #status) @?= 200
               assertBool "Response HTTP headers have not been received properly." $ list_subset (map_to_lower hs) (map_to_lower $ http_response_headers resp)
               check_http_response resp,
             simpleTestCase "maximum number of response headers exceeded" ecid $ \cid -> do
-              {- These 6 response headers are always included:
+              {- These 5 response headers are always included:
                   date: Jan 1 1970 00:00:00 GMT
                   content-type: text/plain; charset=utf-8
                   content-length: 0
-                  connection: close
                   access-control-allow-origin: *
                   access-control-allow-credentials: true
               -}
-              let n = http_headers_max_number - 6 + 1
+              let n = http_headers_max_number - 5 + 1
               ic_http_get_request' (ic00viaWithCyclesRefund 0 cid) sub httpbin_proto ("many_response_headers/" ++ show n) Nothing Nothing cid >>= isReject [1],
             -- "The following additional limits apply to HTTP requests and HTTP responses from the remote sever: the number of bytes representing a header name must not exceed 8KiB."
 

rs/pocket_ic_server/tests/spec_test.rs

@@ -6,13 +6,13 @@ use ic_registry_routing_table::{canister_id_into_u64, CanisterIdRange};
 use ic_registry_subnet_type::SubnetType;
 use pocket_ic::common::rest::DtsFlag;
 use pocket_ic::PocketIcBuilder;
+use rcgen::{CertificateParams, KeyPair};
 use spec_compliance::run_ic_ref_test;
+use std::io::Write;
 use std::process::{Command, Stdio};
 use std::time::Duration;
 use tempfile::NamedTempFile;
 
-const LOCALHOST: &str = "127.0.0.1";
-
 const EXCLUDED: &[&str] = &[
     // we do not enforce https in PocketIC
     "$0 ~ /url must start with https:/",
@@ -53,14 +53,44 @@ fn subnet_config(
     )
 }
 
-fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included_tests: Vec<&str>) {
-    // start httpbin webserver to test canister HTTP outcalls
+// `httpbin_https` can only be set to `true` in a single test
+fn setup_and_run_ic_ref_test(
+    test_nns: bool,
+    httpbin_https: bool,
+    excluded_tests: Vec<&str>,
+    included_tests: Vec<&str>,
+) {
+    // start httpbin webserver to test canister http outcalls
     let httpbin_path = std::env::var_os("HTTPBIN_BIN").expect("Missing httpbin binary path");
     let mut cmd = Command::new(httpbin_path);
     let port_file = NamedTempFile::new().unwrap();
     let port_file_path = port_file.path().to_path_buf();
     cmd.arg("--port-file")
         .arg(port_file_path.as_os_str().to_str().unwrap());
+
+    if httpbin_https {
+        // generate root TLS certificate
+        let root_key_pair = KeyPair::generate().unwrap();
+        let root_cert = CertificateParams::new(vec!["localhost".to_string()])
+            .unwrap()
+            .self_signed(&root_key_pair)
+            .unwrap();
+        let (mut cert_file, cert_path) = NamedTempFile::new().unwrap().keep().unwrap();
+        cert_file.write_all(root_cert.pem().as_bytes()).unwrap();
+        let (mut key_file, key_path) = NamedTempFile::new().unwrap().keep().unwrap();
+        key_file
+            .write_all(root_key_pair.serialize_pem().as_bytes())
+            .unwrap();
+
+        // set `SSL_CERT_FILE` so that the canister http outcalls adapter accepts the self-signed certificate
+        // (this affects all tests and thus `httbin_https` should only be set to `true` in a single test)
+        std::env::set_var("SSL_CERT_FILE", cert_path.clone());
+        std::env::remove_var("NIX_SSL_CERT_FILE");
+
+        cmd.arg("--cert-file").arg(cert_path);
+        cmd.arg("--key-file").arg(key_path);
+    }
+
     cmd.stdout(Stdio::inherit())
         .stderr(Stdio::inherit())
         .spawn()
@@ -73,7 +103,7 @@ fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included
                 .trim_end()
                 .parse()
                 .expect("Failed to parse port to number");
-            break format!("{}:{}", LOCALHOST, port);
+            break format!("localhost:{}", port);
         }
         std::thread::sleep(Duration::from_millis(20));
     };
@@ -151,8 +181,13 @@ fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included
         nns_subnet_config
     };
 
+    let httpbin_proto = if httpbin_https {
+        Some("https://".to_string())
+    } else {
+        Some("http://".to_string())
+    };
     run_ic_ref_test(
-        Some("http://".to_string()),
+        httpbin_proto,
         Some(httpbin_url),
         ic_ref_test_path.into_os_string().into_string().unwrap(),
         ic_test_data_path,
@@ -167,10 +202,20 @@ fn setup_and_run_ic_ref_test(test_nns: bool, excluded_tests: Vec<&str>, included
 
 #[test]
 fn ic_ref_test_nns() {
-    setup_and_run_ic_ref_test(true, EXCLUDED.to_vec(), vec![])
+    setup_and_run_ic_ref_test(true, false, EXCLUDED.to_vec(), vec![])
 }
 
 #[test]
 fn ic_ref_test_app() {
-    setup_and_run_ic_ref_test(false, EXCLUDED.to_vec(), vec![])
+    setup_and_run_ic_ref_test(false, true, EXCLUDED.to_vec(), vec![])
+}
+
+#[test]
+fn ic_ref_test_canister_http() {
+    setup_and_run_ic_ref_test(
+        false,
+        false,
+        EXCLUDED.to_vec(),
+        vec!["$0 ~ /canister http/"],
+    )
 }

rs/tests/httpbin-rs/src/main.rs

@@ -32,13 +32,15 @@ use tokio::{
 use tokio_rustls::TlsAcceptor;
 use tower::Service;
 
-const DETERMINISTIC_HEADERS: [(&str, &str); 4] = [
-    ("Access-Control-Allow-Origin", "*"),
-    ("Access-Control-Allow-Credentials", "true"),
-    ("Connection", "close"),
-    ("Date", "Jan 1 1970 00:00:00 GMT"),
+const DETERMINISTIC_HEADERS: [(&str, &str); 3] = [
+    ("access-control-allow-origin", "*"),
+    ("access-control-allow-credentials", "true"),
+    ("date", "Jan 1 1970 00:00:00 GMT"),
 ];
 
+/// Set a very large limit for the headers to accept.
+const MAX_HEADER_LIST_SIZE: u32 = 1024 * 1024; // 1MiB
+
 /// Returns a normal HTML response
 async fn root_handler() -> Html<&'static str> {
     Html(
@@ -310,6 +312,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                                 }
                             };
                             if let Err(err) = Builder::new(TokioExecutor::new())
+                                .http2()
+                                .max_header_list_size(MAX_HEADER_LIST_SIZE)
                                 .serve_connection(TokioIo::new(tls_stream), hyper_service)
                                 .await
                             {