Provide capability for stable port assignments

This addresses the issue of an instances ports changing on restart
by providing a configuration knob to assign ports to `cmgr`.  If
the `CMGR_PORTS` environment variable is present, `cmgr` will
assume that it has the ability to fully control/assign the ports
in this range without further coordination with the OS or another
process.  This allows it to assign a port number to instances before
creating the instance with Docker which then enables Docker to keep
the same port between restarts of the same container.

Supersedes #31
Fixes #32

README.md

@@ -58,6 +58,14 @@ address is not bound to the host running the Docker daemon, this value gets
 silently ignored by Docker and the exposed ports will be bound to the loopback
 interface.)
 
+- *CMGR\_PORTS*: the range of ports that are dedicated for serving challenges;
+cmgr will assume that it fully owns these ports and nothing else will try
+to use them (i.e., not in ephemeral range or overlapping with a service
+running on the host); format is '1000-1000'.  Ephemeral ports on a Linux host
+can be enumerated with `cat /proc/sys/net/ipv4/ip_local_port_range` and adjusted
+with `sysctl`.  Some programs (e.g., `docker`) will need to be restarted after
+adjusting the kernel parameter.
+
 Additionally, we rely on the Docker SDK's ability to self-configure base off
 environment variables.  The documentation for those variables can be found at
 [https://docs.docker.com/engine/reference/commandline/cli/](https://docs.docker.com/engine/reference/commandline/cli/).

cmd/cmgr/main.go

@@ -219,6 +219,11 @@ Relevant environment variables:
       does not exist on the host running the Docker daemon, Docker will silently
       ignore this value and instead bind to the loopback address
 
+  CMGR_PORTS - the range of ports that are dedicated for serving challenges;
+      cmgr will assume that it fully owns these ports and nothing else will
+      try to use them (i.e., not in ephemeral range or overlapping with a
+      service running on the host); format is '1000-1000'
+
   CMGR_REGISTRY - the host/IP and follow on path for a docker registry; all
       frozen challenges will be pushed as images into this registry (i.e.,
       <registry>/<challenge_slug>).

cmd/cmgrd/main.go

@@ -86,6 +86,11 @@ Relevant environment variables:
       and should be one of the following: debug, info, warn, error, or disabled
       (defaults to 'info')
 
+  CMGR_PORTS - the range of ports that are dedicated for serving challenges;
+      cmgr will assume that it fully owns these ports and nothing else will
+      try to use them (i.e., not in ephemeral range or overlapping with a
+      service running on the host); format is '1000-1000'
+
   CMGR_INTERFACE - the host interface/address to which published challenge
       ports should be bound (defaults to '0.0.0.0'); if the specified interface
       does not exist on the host running the Docker daemon, Docker will silently

cmgr/database.go

@@ -207,6 +207,18 @@ func (m *Manager) getReversePortMap(id ChallengeId) (map[string]string, error) {
 	return rpm, nil
 }
 
+func (m *Manager) usedPortSet() (map[int]struct{}, error) {
+	var ports []int
+	err := m.db.Select(&ports, "SELECT port FROM portAssignments;")
+
+	portSet := make(map[int]struct{})
+	for _, port := range ports {
+		portSet[port] = struct{}{}
+	}
+
+	return portSet, err
+}
+
 func (m *Manager) safeToRefresh(new *ChallengeMetadata) bool {
 	old, err := m.lookupChallengeMetadata(new.Id)
 	if err != nil {

cmgr/docker.go

@@ -12,11 +12,13 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/docker/docker/api/types"
@@ -64,7 +66,76 @@ func (m *Manager) initDocker() error {
 		m.authString = base64.StdEncoding.EncodeToString([]byte(authPayload))
 	}
 
-	return nil
+	m.portLow, m.portHigh, err = getPortRange()
+	if err != nil {
+		m.log.errorf("%s", err)
+	}
+
+	return err
+}
+
+func getPortRange() (int, int, error) {
+	portRange := os.Getenv(PORTS_ENV)
+	if portRange == "" {
+		return 0, 0, nil
+	}
+
+	portStrs := strings.Split(portRange, "-")
+	if len(portStrs) != 2 {
+		return 0, 0, fmt.Errorf("malformed port range: '%s' does not contain '-' character", portRange)
+	}
+
+	var low int
+	var high int
+	var err error
+	low, err = strconv.Atoi(portStrs[0])
+	if err == nil {
+		high, err = strconv.Atoi(portStrs[1])
+	}
+
+	if err != nil {
+		return 0, 0, err
+	}
+
+	if low < 1024 || high > (1<<16) || high < low {
+		err = fmt.Errorf("bad port range: %d-%d either contains invalid/privileged ports or includes 0 ports", low, high)
+	}
+
+	return low, high, err
+}
+
+// Returns a string to simplify integration with Docker's Go API.  Specifically,
+// an empty string will tell it to use an ephemeral port while a non-empty string
+// (even if it is "0") will tell it to attempt binding to that specific port.
+func (m *Manager) getFreePort() (string, error) {
+	if m.portLow == 0 {
+		return "", nil
+	}
+
+	numPorts := m.portHigh - m.portLow + 1
+
+	// Get currently used ports...
+	ports, err := m.usedPortSet()
+	if err != nil {
+		return "", nil
+	}
+
+	// Pick a random starting point in the port range...
+	port := rand.Intn(numPorts) + m.portLow
+
+	// Sweep through ports looking for a free one...
+	for i := 0; i < numPorts; i++ {
+		if _, used := ports[port]; !used {
+			return fmt.Sprint(port), nil
+		}
+
+		port = port + 1
+		if port > m.portHigh {
+			port = m.portLow
+		}
+	}
+
+	return "", fmt.Errorf("All ports between %d and %d are in use", m.portLow, m.portHigh)
 }
 
 func (b *BuildMetadata) makeFlag() *string {
@@ -551,13 +622,23 @@ func (m *Manager) stopNetwork(instance *InstanceMetadata) error {
 	return err
 }
 
-func (m *Manager) startContainers(build *BuildMetadata, instance *InstanceMetadata) error {
+// This approach is pretty heavy-handed and effectively serializes the creation
+// of all challenges that expose ports.  If this becomes a performance issue,
+// may need to look at fully managing ports in memory rather than a hybrid
+// with the SQLite database.
+var portLock sync.Mutex
 
+func (m *Manager) startContainers(build *BuildMetadata, instance *InstanceMetadata) error {
 	revPortMap, err := m.getReversePortMap(build.Challenge)
 	if err != nil {
 		return err
 	}
 
+	if len(revPortMap) != 0 {
+		// No need to lock the port mapping if we are not mapping any ports...
+		portLock.Lock()
+		defer portLock.Unlock()
+	}
 	// Call create in docker
 	netname := instance.getNetworkName()
 	for _, image := range build.Images {
@@ -568,8 +649,14 @@ func (m *Manager) startContainers(build *BuildMetadata, instance *InstanceMetada
 		publishedPorts := nat.PortMap{}
 		for _, portStr := range image.Ports {
 			port := nat.Port(portStr)
+			hostPort, err := m.getFreePort()
+			if err != nil {
+				return err
+			}
 			exposedPorts[port] = struct{}{}
-			publishedPorts[port] = []nat.PortBinding{{HostIP: m.challengeInterface}}
+			publishedPorts[port] = []nat.PortBinding{
+				{HostIP: m.challengeInterface, HostPort: hostPort},
+			}
 		}
 
 		cConfig := container.Config{

cmgr/dockerfiles/seccomp.json

@@ -423,7 +423,7 @@
 					"op": "SCMP_CMP_EQ"
 				}
 			],
-			"comment": "Clear personality flags",
+			"comment": "",
 			"includes": {},
 			"excludes": {}
 		},
@@ -439,7 +439,7 @@
 					"op": "SCMP_CMP_MASKED_EQ"
 				}
 			],
-			"comment": "arg ^ ~(UNAME26 | ADDR_NO_RANDOMIZE | PER_LINUX32) == 0",
+			"comment": "",
 			"includes": {},
 			"excludes": {}
 		},
@@ -455,7 +455,7 @@
 					"op": "SCMP_CMP_EQ"
 				}
 			],
-			"comment": "Query current personality flags",
+			"comment": "",
 			"includes": {},
 			"excludes": {}
 		},

cmgr/types.go

@@ -17,6 +17,7 @@ const (
 	REGISTRY_TOKEN_ENV string = "CMGR_REGISTRY_TOKEN"
 	LOGGING_ENV        string = "CMGR_LOGGING"
 	IFACE_ENV          string = "CMGR_INTERFACE"
+	PORTS_ENV          string = "CMGR_PORTS"
 
 	DYNAMIC_INSTANCES int = -1
 	LOCKED            int = -2
@@ -40,6 +41,8 @@ type Manager struct {
 	challengeInterface   string
 	challengeRegistry    string
 	authString           string
+	portLow              int
+	portHigh             int
 }
 
 type PortInfo struct {