feat(enginenetx): support HTTP and HTTPS proxies

This diff completes the work we have been doing for a few days
now and provides HTTP and HTTPS proxy support, in addition to
SOCKS5 support, for the engine-specific network.

We did this work in the context of ooni/probe#2531
and ooni/probe#1955.

internal/enginenetx/http.go

@@ -48,13 +48,13 @@ func NewHTTPTransport(
 	resolver model.Resolver,
 ) *HTTPTransport {
 	dialer := netxlite.NewDialerWithResolver(logger, resolver)
-	dialer = netxlite.MaybeWrapWithProxyDialer(dialer, proxyURL)
 	handshaker := netxlite.NewTLSHandshakerStdlib(logger)
 	tlsDialer := netxlite.NewTLSDialer(dialer, handshaker)
-	// TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport
-	// function, but we can probably avoid using it, given that this code is
-	// not using tracing and does not care about those quirks.
-	txp := netxlite.NewHTTPTransport(logger, dialer, tlsDialer)
+	txp := netxlite.NewHTTPTransportWithOptions(
+		logger, dialer, tlsDialer,
+		netxlite.HTTPTransportOptionDisableCompression(false),
+		netxlite.HTTPTransportOptionProxyURL(proxyURL), // nil implies "no proxy"
+	)
 	txp = bytecounter.WrapHTTPTransport(txp, counter)
 	return &HTTPTransport{txp}
 }

internal/enginenetx/http_test.go

@@ -1,34 +1,263 @@
-package enginenetx
+package enginenetx_test
 
 import (
+	"context"
+	"net"
+	"net/http"
+	"net/url"
 	"testing"
+	"time"
 
+	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/bytecounter"
+	"github.com/ooni/probe-cli/v3/internal/enginenetx"
+	"github.com/ooni/probe-cli/v3/internal/measurexlite"
 	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/netemx"
 	"github.com/ooni/probe-cli/v3/internal/netxlite"
+	"github.com/ooni/probe-cli/v3/internal/testingsocks5"
+	"github.com/ooni/probe-cli/v3/internal/testingx"
 )
 
-func TestHTTPTransport(t *testing.T) {
+func TestHTTPTransportWAI(t *testing.T) {
+	t.Run("is WAI when not using any proxy", func(t *testing.T) {
+		env := netemx.MustNewScenario(netemx.InternetScenario)
+		defer env.Close()
 
-	// TODO(bassosimone): we should replace this integration test with netemx
-	// as soon as we can sever the hard link between netxlite and this pkg
-	t.Run("is working as intended", func(t *testing.T) {
-		txp := NewHTTPTransport(
-			bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
-		client := txp.NewHTTPClient()
-		resp, err := client.Get("https://www.google.com/robots.txt")
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer resp.Body.Close()
-		if resp.StatusCode != 200 {
-			t.Fatal("unexpected status code")
-		}
+		env.Do(func() {
+			txp := enginenetx.NewHTTPTransport(
+				bytecounter.New(),
+				model.DiscardLogger,
+				nil,
+				netxlite.NewStdlibResolver(model.DiscardLogger),
+			)
+			client := txp.NewHTTPClient()
+			resp, err := client.Get("https://www.example.com/")
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("%+v", resp)
+			defer resp.Body.Close()
+			if resp.StatusCode != 200 {
+				t.Fatal("unexpected status code")
+			}
+		})
+	})
+
+	t.Run("is WAI when using a SOCKS5 proxy", func(t *testing.T) {
+		// create internet measurement scenario
+		env := netemx.MustNewScenario(netemx.InternetScenario)
+		defer env.Close()
+
+		// create a proxy using the client's TCP/IP stack
+		proxy := testingsocks5.MustNewServer(
+			log.Log,
+			&netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}},
+			&net.TCPAddr{
+				IP:   net.ParseIP(env.ClientStack.IPAddress()),
+				Port: 9050,
+			},
+		)
+		defer proxy.Close()
+
+		env.Do(func() {
+			txp := enginenetx.NewHTTPTransport(
+				bytecounter.New(),
+				model.DiscardLogger,
+				&url.URL{
+					Scheme: "socks5",
+					Host:   net.JoinHostPort(env.ClientStack.IPAddress(), "9050"),
+					Path:   "/",
+				},
+				netxlite.NewStdlibResolver(model.DiscardLogger),
+			)
+			client := txp.NewHTTPClient()
+
+			// To make sure we're connecting to the expected endpoint, we're going to use
+			// measurexlite and tracing to observe the destination endpoints
+			trace := measurexlite.NewTrace(0, time.Now())
+			ctx := netxlite.ContextWithTrace(context.Background(), trace)
+
+			// create request using the above context
+			//
+			// Implementation note: we cannot use HTTPS with netem here as explained
+			// by the https://github.com/ooni/probe/issues/2536 issue.
+			req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			resp, err := client.Do(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("%+v", resp)
+			defer resp.Body.Close()
+			if resp.StatusCode != 200 {
+				t.Fatal("unexpected status code")
+			}
+
+			// make sure that we only connected to the SOCKS5 proxy
+			tcpConnects := trace.TCPConnects()
+			if len(tcpConnects) <= 0 {
+				t.Fatal("expected at least one TCP connect")
+			}
+			for idx, entry := range tcpConnects {
+				t.Logf("%d: %+v", idx, entry)
+				if entry.IP != env.ClientStack.IPAddress() {
+					t.Fatal("unexpected IP address")
+				}
+				if entry.Port != 9050 {
+					t.Fatal("unexpected port")
+				}
+			}
+		})
+	})
+
+	t.Run("is WAI when using an HTTP proxy", func(t *testing.T) {
+		// create internet measurement scenario
+		env := netemx.MustNewScenario(netemx.InternetScenario)
+		defer env.Close()
+
+		// create a proxy using the client's TCP/IP stack
+		proxy := testingx.MustNewHTTPServerEx(
+			&net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 8080},
+			env.ClientStack,
+			testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
+				Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}),
+		)
+		defer proxy.Close()
+
+		env.Do(func() {
+			txp := enginenetx.NewHTTPTransport(
+				bytecounter.New(),
+				model.DiscardLogger,
+				&url.URL{
+					Scheme: "http",
+					Host:   net.JoinHostPort(env.ClientStack.IPAddress(), "8080"),
+					Path:   "/",
+				},
+				netxlite.NewStdlibResolver(model.DiscardLogger),
+			)
+			client := txp.NewHTTPClient()
+
+			// To make sure we're connecting to the expected endpoint, we're going to use
+			// measurexlite and tracing to observe the destination endpoints
+			trace := measurexlite.NewTrace(0, time.Now())
+			ctx := netxlite.ContextWithTrace(context.Background(), trace)
+
+			// create request using the above context
+			//
+			// Implementation note: we cannot use HTTPS with netem here as explained
+			// by the https://github.com/ooni/probe/issues/2536 issue.
+			req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			resp, err := client.Do(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("%+v", resp)
+			defer resp.Body.Close()
+			if resp.StatusCode != 200 {
+				t.Fatal("unexpected status code")
+			}
+
+			// make sure that we only connected to the SOCKS5 proxy
+			tcpConnects := trace.TCPConnects()
+			if len(tcpConnects) <= 0 {
+				t.Fatal("expected at least one TCP connect")
+			}
+			for idx, entry := range tcpConnects {
+				t.Logf("%d: %+v", idx, entry)
+				if entry.IP != env.ClientStack.IPAddress() {
+					t.Fatal("unexpected IP address")
+				}
+				if entry.Port != 8080 {
+					t.Fatal("unexpected port")
+				}
+			}
+		})
+	})
+
+	t.Run("is WAI when using an HTTPS proxy", func(t *testing.T) {
+		// create internet measurement scenario
+		env := netemx.MustNewScenario(netemx.InternetScenario)
+		defer env.Close()
+
+		// create a proxy using the client's TCP/IP stack
+		proxy := testingx.MustNewHTTPServerTLSEx(
+			&net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 4443},
+			env.ClientStack,
+			testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
+				Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}),
+			env.ClientStack,
+		)
+		defer proxy.Close()
+
+		env.Do(func() {
+			txp := enginenetx.NewHTTPTransport(
+				bytecounter.New(),
+				model.DiscardLogger,
+				&url.URL{
+					Scheme: "https",
+					Host:   net.JoinHostPort(env.ClientStack.IPAddress(), "4443"),
+					Path:   "/",
+				},
+				netxlite.NewStdlibResolver(model.DiscardLogger),
+			)
+			client := txp.NewHTTPClient()
+
+			// To make sure we're connecting to the expected endpoint, we're going to use
+			// measurexlite and tracing to observe the destination endpoints
+			trace := measurexlite.NewTrace(0, time.Now())
+			ctx := netxlite.ContextWithTrace(context.Background(), trace)
+
+			// create request using the above context
+			//
+			// Implementation note: we cannot use HTTPS with netem here as explained
+			// by the https://github.com/ooni/probe/issues/2536 issue.
+			req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			resp, err := client.Do(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Logf("%+v", resp)
+			defer resp.Body.Close()
+			if resp.StatusCode != 200 {
+				t.Fatal("unexpected status code")
+			}
+
+			// make sure that we only connected to the SOCKS5 proxy
+			tcpConnects := trace.TCPConnects()
+			if len(tcpConnects) <= 0 {
+				t.Fatal("expected at least one TCP connect")
+			}
+			for idx, entry := range tcpConnects {
+				t.Logf("%d: %+v", idx, entry)
+				if entry.IP != env.ClientStack.IPAddress() {
+					t.Fatal("unexpected IP address")
+				}
+				if entry.Port != 4443 {
+					t.Fatal("unexpected port")
+				}
+			}
+		})
 	})
 
 	t.Run("NewHTTPClient returns a client with a cookie jar", func(t *testing.T) {
-		txp := NewHTTPTransport(
-			bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
+		txp := enginenetx.NewHTTPTransport(
+			bytecounter.New(),
+			model.DiscardLogger,
+			nil,
+			netxlite.NewStdlibResolver(model.DiscardLogger),
+		)
 		client := txp.NewHTTPClient()
 		if client.Jar == nil {
 			t.Fatal("expected non-nil cookie jar")

internal/netemx/qaenv.go

@@ -140,8 +140,8 @@ type QAEnv struct {
 	// clientNICWrapper is the OPTIONAL wrapper for the client NIC.
 	clientNICWrapper netem.LinkNICWrapper
 
-	// clientStack is the client stack to use.
-	clientStack *netem.UNetStack
+	// ClientStack is the client stack to use.
+	ClientStack *netem.UNetStack
 
 	// closables contains all entities where we have to take care of closing.
 	closables []io.Closer
@@ -197,7 +197,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv {
 	env := &QAEnv{
 		baseLogger:                config.logger,
 		clientNICWrapper:          config.clientNICWrapper,
-		clientStack:               nil,
+		ClientStack:               nil,
 		closables:                 []io.Closer{},
 		emulateAndroidGetaddrinfo: &atomic.Bool{},
 		ispResolverConfig:         netem.NewDNSConfig(),
@@ -208,7 +208,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv {
 	}
 
 	// create all the required internals
-	env.clientStack = env.mustNewClientStack(config)
+	env.ClientStack = env.mustNewClientStack(config)
 	env.closables = append(env.closables, env.mustNewNetStacks(config)...)
 
 	return env
@@ -306,7 +306,7 @@ func (env *QAEnv) EmulateAndroidGetaddrinfo(value bool) {
 // Do executes the given function such that [netxlite] code uses the
 // underlying clientStack rather than ordinary networking code.
 func (env *QAEnv) Do(function func()) {
-	var stack netem.UnderlyingNetwork = env.clientStack
+	var stack netem.UnderlyingNetwork = env.ClientStack
 	if env.emulateAndroidGetaddrinfo.Load() {
 		stack = &androidStack{stack}
 	}

internal/netxlite/maybeproxy.go

@@ -22,6 +22,8 @@ type proxyDialer struct {
 
 // MaybeWrapWithProxyDialer returns the original dialer if the proxyURL is nil
 // and otherwise returns a wrapped dialer that implements proxying.
+//
+// Deprecated: do not use this function in new code.
 func MaybeWrapWithProxyDialer(dialer model.Dialer, proxyURL *url.URL) model.Dialer {
 	if proxyURL == nil {
 		return dialer