add prometheus and counter for testing

Our statsd integration with prometheus has been turning counters that
should be 1 into 0. It's unclear why this is happening and this patch is
an attempt to debug the issue further.

The foobar_test counter added here is to help determine why. It's
incremented once on boot in both the prometheus and in statsd. We'll
determine if the problem is in prometheus, grafana, telegraf, or in our
statsd client (unlikely this last one) by emitting in both and seeing
how they're captured.

The rest of the org is standardizing on prometheus for metrics up and
down the stack, so we should do the same, anyway. We don't commit to
migrating all of the existing metrics here, but make it plausible to do
so.

This adds the port 2112 to Dockerfile for autograph as the place where
the prometheus metrics are served from.

Follow up patches in our infra-as-code will add the fetching of
prometheus metrics.

Updates AUT-393
Updates AUT-199

Dockerfile

@@ -64,6 +64,7 @@ RUN cd /app/src/autograph/tools/makecsr && go build -o /go/bin/makecsr .
 #------------------------------------------------------------------------------
 FROM prebuild
 EXPOSE 8000
+EXPOSE 2112
 
 # Copy compiled appliation from the builder.
 ADD . /app/src/autograph

autograph.yaml

@@ -11,6 +11,9 @@ statsd:
     namespace: "autograph."
     buflen: 1
 
+debugserver:
+    listen: "0.0.0.0:2112"
+
 # When using an HSM, place the path to the pkcs11 client library,
 # token label and pin of the crypto user in the configuration below
 # hsm:

docker-compose.yml

@@ -27,6 +27,7 @@ services:
       - db
     ports:
       - "8000:8000"
+      - "2112:2112"
     volumes:
       - apptmpdir:/tmp/
     healthcheck:
@@ -116,7 +117,4 @@ services:
       - db
     user: "0"
     working_dir: "/app/src/autograph/"
-    command:
-      [
-        "./bin/run_unit_tests.sh"
-      ]
+    command: ["./bin/run_unit_tests.sh"]

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/lib/pq v1.10.9
 	github.com/miekg/pkcs11 v1.0.3
 	github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c
+	github.com/prometheus/client_golang v1.20.5
 	github.com/sirupsen/logrus v1.9.3
 	github.com/youtube/vitess v2.1.1+incompatible
 	go.mozilla.org/cose v0.0.0-20200221144611-2ea72a6b3de3
@@ -66,6 +67,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 // indirect
 	github.com/aws/smithy-go v1.22.1 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
@@ -98,15 +100,20 @@ require (
 	github.com/hashicorp/go-sockaddr v1.0.7 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/vault/api v1.15.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/ugorji/go v0.0.0-20180112141927-9831f2c3ac10 // indirect

go.sum

@@ -106,6 +106,8 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 h1:VwhTrsTuVn52an4mXx29PqRzs2Dv
 github.com/aws/aws-sdk-go-v2/service/sts v1.33.6/go.mod h1:+8h7PZb3yY5ftmVLD7ocEoE98hdc8PoKS0H3wfx1dlc=
 github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro=
 github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -216,6 +218,8 @@ github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/
 github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8=
 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -245,6 +249,8 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c h1:yE1NxRAZA3wF0laDWECtOe2J0tFjSHUI6MXXbMif+QY=
 github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c/go.mod h1:Is/Ucts/yU/mWyGR8yELRoO46mejouKsJfQLAIfTR18=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
@@ -262,6 +268,14 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm
 github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
 github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
 github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=

main.go

@@ -43,6 +43,7 @@ import (
 
 	"github.com/DataDog/datadog-go/v5/statsd"
 	"github.com/mozilla-services/autograph/crypto11"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // configuration loads a yaml file that contains the configuration of Autograph
@@ -59,6 +60,11 @@ type configuration struct {
 		Namespace string
 		Buflen    int
 	}
+
+	// DebugServer are the settings for the control plane HTTP server where
+	// metrics are exposed for collection and some limited utitilites can live.
+	DebugServer debugServerConfig `yaml:"debugserver"`
+
 	HSM                   crypto11.PKCS11Config
 	Database              database.Config
 	Signers               []signer.Configuration
@@ -69,6 +75,12 @@ type configuration struct {
 	MonitorInterval       time.Duration
 }
 
+type debugServerConfig struct {
+	// Listen is a TCP address for the HTTP server to listen on. See
+	// http.Server's Addr field for details.
+	Listen string `yaml:"listen"`
+}
+
 // An autographer is a running instance of an autograph service,
 // with all signers and permissions configured
 type autographer struct {
@@ -240,6 +252,29 @@ func run(conf configuration, listen string, debug bool) {
 		router.PathPrefix(prefix).Handler(http.StripPrefix(prefix, http.FileServer(http.Dir(parsedURL.Path))))
 	}
 
+	ag.stats.Incr(foobarTestCounterName, nil, 1)
+	foobarTestCounter.Inc()
+
+	if conf.DebugServer.Listen != "" {
+		log.Infof("starting debug server on %s", conf.DebugServer.Listen)
+		go func() {
+			mux := mux.NewRouter()
+			mux.Handle("/metrics", promhttp.Handler())
+			srv := http.Server{
+				Addr:         conf.DebugServer.Listen,
+				IdleTimeout:  conf.Server.IdleTimeout,
+				ReadTimeout:  conf.Server.ReadTimeout,
+				WriteTimeout: conf.Server.WriteTimeout,
+				Handler:      mux,
+			}
+
+			err := srv.ListenAndServe()
+			if err != nil {
+				log.Fatalf("unable to start up debug server on %s: %s", conf.DebugServer.Listen, err)
+			}
+		}()
+	}
+
 	server := &http.Server{
 		IdleTimeout:  conf.Server.IdleTimeout,
 		ReadTimeout:  conf.Server.ReadTimeout,

stats.go

@@ -6,9 +6,21 @@ import (
 	"sync/atomic"
 
 	"github.com/DataDog/datadog-go/v5/statsd"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	log "github.com/sirupsen/logrus"
 )
 
+var (
+	// TODO(AUT-393): remove the statsd and prometheus counter using this name
+	// once we're done testing.
+	foobarTestCounterName = "foobar_test"
+	foobarTestCounter     = promauto.NewCounter(prometheus.CounterOpts{
+		Name: foobarTestCounterName,
+		Help: "A counter used for testing how prometheus and statsd metrics differ",
+	})
+)
+
 func loadStatsd(conf configuration) (*statsd.Client, error) {
 	statsdClient, err := statsd.New(conf.Statsd.Addr, statsd.WithNamespace(conf.Statsd.Namespace))
 	if err != nil {