AUT-54: Add endpoint to serve local x5u chains

bin/run_integration_tests.sh

@@ -25,8 +25,8 @@ while test "true" != "$(docker inspect -f {{.State.Running}} autograph-app-hsm)"
 done
 
 # fetch the updated root hash from the app-hsm service
-docker cp autograph-app-hsm:/tmp/normandy_dev_root_hash.txt .
-APP_HSM_NORMANDY_ROOT_HASH=$(grep '[0-9A-F]' normandy_dev_root_hash.txt | tr -d '\r\n')
+APP_HSM_NORMANDY_ROOT_HASH=$(docker compose exec app-hsm yq -r '.signers[] | select(.id == "normandy").cacert' /app/autograph.softhsm.yaml | \
+                             openssl x509 -outform DER | sha256sum | awk '{print $1}')
 
 # start the monitor lambda emulators
 echo "checking autograph monitors"

docker-compose.yml

@@ -87,8 +87,6 @@ services:
       - app
     depends_on:
       - app
-    volumes:
-      - apptmpdir:/tmp/
 
   monitor-hsm-lambda-emulator:
     container_name: autograph-monitor-hsm-lambda-emulator
@@ -109,8 +107,6 @@ services:
       - app-hsm
     depends_on:
       - app-hsm
-    volumes:
-      - hsmtmpdir:/tmp/
 
   unit-test:
     container_name: autograph-unit-test

docs/endpoints.md

@@ -271,6 +271,16 @@ See `/sign/data`, the response format is identical.
 A successful request return a `201 Created` with a response
 body containing an S/MIME detached signature encoded with Base 64.
 
+## /x5u/:keyid/
+
+This is an endpoint used to fetch certificate chains which are generated and
+stored locally. If the signer is configured with an `X5U` using the `file://`
+scheme, then the contents of that file location are served under this path.
+
+This path is only intended to simplify local development and testing. Production
+signers should store their X5U certificate chains with a cloud storage provider
+such as Amazon S3 or Google Cloud Storage.
+
 ## /\_\_monitor\_\_
 
 This is a special endpoint designed to monitor the status of all signers

handlers.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path"
 	"time"
@@ -74,6 +75,27 @@ func logSigningRequestFailure(sigreq formats.SignatureRequest, sigresp formats.S
 	}).Info(fmt.Sprintf("signing operation failed with error: %v", err))
 }
 
+// rewriteLocalX5U checks for X5U certificate chains using the `file://` scheme
+// and rewrites them to use the `/x5u/:keyid/` endpoint instead, which should
+// mirror the contents of the signer's X5U location, and returns the updated URL.
+//
+// If the X5U certificate chain uses any other scheme, then the original URL is returned
+// without change.
+func rewriteLocalX5U(r *http.Request, keyid string, x5u string) string {
+	parsedX5U, err := url.Parse(x5u)
+	if err == nil && parsedX5U.Scheme == "file" {
+		newX5U := url.URL{
+			Scheme: "http",
+			Host:   r.Host,
+			Path:   path.Join("x5u", keyid, path.Base(parsedX5U.Path)),
+		}
+		return newX5U.String()
+	}
+
+	// Otherwise, return the X5U unmodified
+	return x5u
+}
+
 // handleSignature endpoint accepts a list of signature requests in a HAWK authenticated POST request
 // and calls the signers to generate signature responses.
 func (a *autographer) handleSignature(w http.ResponseWriter, r *http.Request) {
@@ -200,9 +222,10 @@ func (a *autographer) handleSignature(w http.ResponseWriter, r *http.Request) {
 			SignerID:   requestedSignerConfig.ID,
 			PublicKey:  requestedSignerConfig.PublicKey,
 			SignedFile: base64.StdEncoding.EncodeToString(signedfile),
-			X5U:        requestedSignerConfig.X5U,
+			X5U:        rewriteLocalX5U(r, requestedSignerConfig.ID, requestedSignerConfig.X5U),
 			SignerOpts: requestedSignerConfig.SignerOpts,
 		}
+
 		// Make sure the signer implements the right interface, then sign the data
 		switch r.URL.RequestURI() {
 		case "/sign/hash":

handlers_test.go

@@ -37,6 +37,96 @@ import (
 	margo "go.mozilla.org/mar"
 )
 
+type HandlerTestCase struct {
+	name   string
+	method string
+	url    string
+
+	// urlRouteVars are https://pkg.go.dev/github.com/gorilla/mux#Vars
+	// as configured with the handler at /config/{keyid:[a-zA-Z0-9-_]{1,64}}
+	// there should only be a keyid var and it should match the url value
+	urlRouteVars map[string]string
+
+	// headers are additional http headers to set
+	headers *http.Header
+
+	// user/auth ID to build an Authorization header for
+	authorizeID string
+	nilBody     bool
+	body        string
+
+	expectedStatus  int
+	expectedHeaders http.Header
+	expectedBody    string
+}
+
+func (testcase *HandlerTestCase) NewRequest(t *testing.T) *http.Request {
+	// test request setup
+	var (
+		req *http.Request
+		err error
+	)
+	if testcase.nilBody {
+		req, err = http.NewRequest(testcase.method, testcase.url, nil)
+	} else {
+		req, err = http.NewRequest(testcase.method, testcase.url, strings.NewReader(testcase.body))
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	req = mux.SetURLVars(req, testcase.urlRouteVars)
+	if testcase.headers != nil {
+		req.Header = *testcase.headers
+	}
+
+	if testcase.authorizeID != "" {
+		auth, err := ag.getAuthByID(testcase.authorizeID)
+		if err != nil {
+			t.Fatal(err)
+		}
+		// getAuthHeader requires a content type and body
+		req.Header.Set("Authorization", hawk.NewRequestAuth(req,
+			&hawk.Credentials{
+				ID:   auth.ID,
+				Key:  auth.Key,
+				Hash: sha256.New},
+			0).RequestHeader())
+	}
+
+	return req
+}
+
+func (testcase *HandlerTestCase) ValidateResponse(t *testing.T, w *httptest.ResponseRecorder) {
+	if w.Code != testcase.expectedStatus {
+		t.Fatalf("test case %s: got code %d but expected %d",
+			testcase.name, w.Code, testcase.expectedStatus)
+	}
+	if w.Body.String() != testcase.expectedBody {
+		t.Fatalf("test case %s: got body %q expected %q", testcase.name, w.Body.String(), testcase.expectedBody)
+	}
+	for expectedHeader, expectedHeaderVals := range testcase.expectedHeaders {
+		vals, ok := w.Header()[expectedHeader]
+		if !ok {
+			t.Fatalf("test case %s: expected header %q not found", testcase.name, expectedHeader)
+		}
+		if strings.Join(vals, "") != strings.Join(expectedHeaderVals, "") {
+			t.Fatalf("test case %s: header vals %q did not match expected %q ", testcase.name, vals, expectedHeaderVals)
+		}
+	}
+}
+
+func (testcase *HandlerTestCase) Run(t *testing.T, handler func(http.ResponseWriter, *http.Request)) {
+	// test request setup
+	var req = testcase.NewRequest(t)
+
+	// run the request
+	w := httptest.NewRecorder()
+	handler(w, req)
+
+	// validate response
+	testcase.ValidateResponse(t, w)
+}
+
 func TestBadRequest(t *testing.T) {
 	t.Parallel()
 
@@ -575,28 +665,7 @@ func TestHandleGetAuthKeyIDs(t *testing.T) {
 
 	const autographDevAliceKeyIDsJSON = "[\"apk_cert_with_ecdsa_sha256\",\"apk_cert_with_ecdsa_sha256_v3\",\"appkey1\",\"appkey2\",\"dummyrsa\",\"dummyrsapss\",\"extensions-ecdsa\",\"extensions-ecdsa-expired-chain\",\"legacy_apk_with_rsa\",\"normandy\",\"pgpsubkey\",\"pgpsubkey-debsign\",\"randompgp\",\"randompgp-debsign\",\"remote-settings\",\"testapp-android\",\"testapp-android-legacy\",\"testapp-android-v3\",\"testauthenticode\",\"testmar\",\"testmarecdsa\",\"webextensions-rsa\",\"webextensions-rsa-with-recommendation\"]"
 
-	var testcases = []struct {
-		name   string
-		method string
-		url    string
-
-		// urlRouteVars are https://pkg.go.dev/github.com/gorilla/mux#Vars
-		// as configured with the handler at /auths/{auth_id:[a-zA-Z0-9-_]{1,255}}/keyids
-		// there should only be an auth_id var and it should match the url value
-		urlRouteVars map[string]string
-
-		// headers are additional http headers to set
-		headers *http.Header
-
-		// user/auth ID to build an Authorization header for
-		authorizeID string
-		nilBody     bool
-		body        string
-
-		expectedStatus  int
-		expectedHeaders http.Header
-		expectedBody    string
-	}{
+	var testcases = []HandlerTestCase{
 		{
 			name:            "invalid method POST returns 405",
 			method:          "POST",
@@ -720,60 +789,8 @@ func TestHandleGetAuthKeyIDs(t *testing.T) {
 			expectedHeaders: http.Header{"Content-Type": []string{"application/json"}},
 		},
 	}
-	for i, testcase := range testcases {
-		// test request setup
-		var (
-			req *http.Request
-			err error
-		)
-		if testcase.nilBody {
-			req, err = http.NewRequest(testcase.method, testcase.url, nil)
-		} else {
-			req, err = http.NewRequest(testcase.method, testcase.url, strings.NewReader(testcase.body))
-		}
-		if err != nil {
-			t.Fatal(err)
-		}
-		req = mux.SetURLVars(req, testcase.urlRouteVars)
-
-		if testcase.headers != nil {
-			req.Header = *testcase.headers
-		}
-		if testcase.authorizeID != "" {
-			auth, err := ag.getAuthByID(testcase.authorizeID)
-			if err != nil {
-				t.Fatal(err)
-			}
-			// getAuthHeader requires a content type and body
-			req.Header.Set("Authorization", hawk.NewRequestAuth(req,
-				&hawk.Credentials{
-					ID:   auth.ID,
-					Key:  auth.Key,
-					Hash: sha256.New},
-				0).RequestHeader())
-		}
-
-		// run the request
-		w := httptest.NewRecorder()
-		ag.handleGetAuthKeyIDs(w, req)
-
-		// validate response
-		if w.Code != testcase.expectedStatus {
-			t.Fatalf("test case %s (%d): got code %d but expected %d",
-				testcase.name, i, w.Code, testcase.expectedStatus)
-		}
-		if w.Body.String() != testcase.expectedBody {
-			t.Fatalf("test case %s (%d): got body %q expected %q", testcase.name, i, w.Body.String(), testcase.expectedBody)
-		}
-		for expectedHeader, expectedHeaderVals := range testcase.expectedHeaders {
-			vals, ok := w.Header()[expectedHeader]
-			if !ok {
-				t.Fatalf("test case %s (%d): expected header %q not found", testcase.name, i, expectedHeader)
-			}
-			if strings.Join(vals, "") != strings.Join(expectedHeaderVals, "") {
-				t.Fatalf("test case %s (%d): header vals %q did not match expected %q ", testcase.name, i, vals, expectedHeaderVals)
-			}
-		}
+	for _, testcase := range testcases {
+		testcase.Run(t, ag.handleGetAuthKeyIDs)
 	}
 }
 

main.go

@@ -13,6 +13,7 @@ import (
 	"flag"
 	"fmt"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"regexp"
@@ -223,6 +224,19 @@ func run(conf configuration, listen string, debug bool) {
 		log.Infof("enabled HTTP perf profiler")
 	}
 
+	// For each signer with a local chain upload location (eg: using the file
+	// scheme) create an handler to serve that directory at the path /x5u/keyid/
+	for _, signerConf := range conf.Signers {
+		parsedURL, err := url.Parse(signerConf.X5U)
+		if err != nil || parsedURL.Scheme != "file" {
+			// This signer doesn't upload certificate chains to local storage.
+			continue
+		}
+
+		prefix := fmt.Sprintf("/x5u/%s/", signerConf.ID)
+		router.PathPrefix(prefix).Handler(http.StripPrefix(prefix, http.FileServer(http.Dir(parsedURL.Path))))
+	}
+
 	server := &http.Server{
 		IdleTimeout:  conf.Server.IdleTimeout,
 		ReadTimeout:  conf.Server.ReadTimeout,

monitor_handler.go

@@ -47,6 +47,7 @@ func (m *monitor) handleMonitor(w http.ResponseWriter, r *http.Request) {
 
 	enc := json.NewEncoder(w)
 	for _, response := range m.sigresps {
+		response.X5U = rewriteLocalX5U(r, response.SignerID, response.X5U)
 		if err := enc.Encode(&response); err != nil {
 			httpError(w, r, http.StatusInternalServerError, "encoding failed with error: %v", err)
 			return

monitor_handler_racing_test.go

@@ -8,9 +8,12 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
+	"os"
 	"strings"
 	"testing"
 
@@ -28,6 +31,43 @@ import (
 
 const autographDevRootHash = `5E:36:F2:14:DE:82:3F:8B:29:96:89:23:5F:03:41:AC:AF:A0:75:AF:82:CB:4C:D4:30:7C:3D:B3:43:39:2A:FE`
 
+func getLocalX5U(x5u string) (body []byte, err error) {
+	parsed, err := url.Parse(x5u)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the URL can be parsed into the form /x5u/keyid/chainfilename then
+	// try to read it directly off the disk, since this is likely a local file
+	segs := strings.Split(parsed.Path, "/")
+	if (len(segs) < 3) || (segs[len(segs)-3] != "x5u") {
+		return nil, fmt.Errorf("x5u URL '%s' does not appear to be locally hosted", x5u)
+	}
+
+	// Find the signer matching the keyid URL segment.
+	keyid := segs[len(segs)-2]
+	for _, s := range ag.getSigners() {
+		config := s.Config()
+		if config.ID != keyid {
+			continue
+		}
+
+		// If the X5U location uses the `file` scheme, read the chain off disk.
+		parsedX5U, err := url.Parse(config.X5U)
+		if err != nil {
+			return nil, err
+		}
+		if parsedX5U.Scheme != "file" {
+			break
+		}
+		return os.ReadFile(parsedX5U.Path)
+	}
+
+	// Otherwise, we would need to perform an HTTP request to wherever this
+	// chain is hosted, but under unit testing there are no such signers.
+	return nil, fmt.Errorf("unable to read x5u from '%s'", x5u)
+}
+
 func TestMonitorPass(t *testing.T) {
 	t.Parallel()
 
@@ -64,7 +104,7 @@ func TestMonitorPass(t *testing.T) {
 				t.Fatalf("verification of monitoring response failed: %v", err)
 			}
 		case contentsignaturepki.Type:
-			body, _, err := contentsignaturepki.GetX5U(&http.Client{}, response.X5U)
+			body, err := getLocalX5U(response.X5U)
 			if err != nil {
 				t.Fatal(err)
 			}