Switch to docker/make; add pytests

.github/workflows/docker-publish.yml

@@ -62,4 +62,3 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
             #cache-from: type=gha
             #cache-to: type=gha,mode=max
-

.github/workflows/tests.yml

@@ -0,0 +1,22 @@
+name: Pytest Unit Tests
+
+on: [push]
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        name: Checkout code
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Build the Project
+        run: make init build
+      - name: Start WEI
+        run: make start
+      - name: Test with pytest
+        run: make test
+      - name: Cleanup
+        run: make remove

.gitignore

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# WEI
+.wei

.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-yaml
+      - id: check-json
+      - id: check-toml
+      - id: check-ast
+      - id: check-merge-conflict
+      - id: check-added-large-files
+      - id: mixed-line-ending
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.6.1
+    hooks:
+      - id: nbstripout
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.1.10
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format

Dockerfile

@@ -4,15 +4,31 @@ LABEL org.opencontainers.image.source=https://github.com/AD-SDL/webcam_module
 LABEL org.opencontainers.image.description="An example module that implements a simple webcam snapshot action"
 LABEL org.opencontainers.image.licenses=MIT
 
-USER wei
-WORKDIR /home/wei
+ARG USER_ID=1000
+ARG GROUP_ID=1000
+ARG CONTAINER_USER=app
+ARG CONTAINER_GROUP=app
+USER ${CONTAINER_USER}
+WORKDIR /home/${CONTAINER_USER}
+
+# Add python packages to path
+ENV PATH="$PATH:/home/${CONTAINER_USER}/.local/bin"
+
+#########################################
+# Module specific logic goes below here #
+#########################################
 
 RUN mkdir -p webcam_module
 
-COPY --chown=wei:wei ./src webcam_module/src
-COPY --chown=wei:wei ./README.md webcam_module/README.md
-COPY --chown=wei:wei ./pyproject.toml webcam_module/pyproject.toml
+COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} ./src webcam_module/src
+COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} ./README.md webcam_module/README.md
+COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} ./pyproject.toml webcam_module/pyproject.toml
+COPY --chown=${CONTAINER_USER}:${CONTAINER_GROUP} ./tests webcam_module/tests
 
-RUN pip install ./webcam_module
+RUN --mount=type=cache,target=/home/${CONTAINER_USER}/.cache,uid=${USER_ID},gid=${GROUP_ID} \
+    pip install -e ./webcam_module
 
 CMD ["python", "webcam_module/src/webcam_rest_node.py"]
+
+#########################################
+USER root

Makefile

@@ -0,0 +1,25 @@
+################################################################################
+# AD-SDL WEI Template Makefile
+################################################################################
+MAKEFILE := $(lastword $(MAKEFILE_LIST))
+MAKEFILE_DIR := $(dir $(MAKEFILE))
+INCLUDE_DIR := $(MAKEFILE_DIR)/make
+
+include $(INCLUDE_DIR)/boilerplate.mk # Boilerplate, can probably leave as-is
+include $(INCLUDE_DIR)/config.mk # Project-specific configuration
+include $(INCLUDE_DIR)/docker.mk # Docker-related rules
+
+################################################################################
+# Rules: Add anything you want to be able to run with `make <target>` below
+
+checks: # Runs all the pre-commit checks
+	@pre-commit install
+	@pre-commit run --all-files || { echo "Checking fixes\n" ; pre-commit run --all-files; }
+
+test: init # Runs all the tests
+	docker compose -f $(COMPOSE_FILE) exec -u app $(APP_NAME) pytest ${MODULE_NAME} $(args)
+
+################################################################################
+
+# Determine which rules don't correspond to actual files (add rules to NOT_PHONY to exclude)
+.PHONY: $(filter-out $(NOT_PHONY), $(RULES))

compose.yaml

@@ -1,14 +1,64 @@
-version: "3.0"
 name: webcam_module
 services:
-  webcam_node:
-    image: ghcr.io/ad-sdl/webcam_module
+  ###########
+  # Modules #
+  ###########
+  webcam_module:
+    container_name: webcam_module
+    image: ${IMAGE}
     build:
       context: .
       dockerfile: Dockerfile
-    command: python /webcam_module/src/webcam_rest_node.py
+    command: python webcam_module/src/webcam_rest_node.py --port 2000
+    env_file: .env
     privileged: true
     volumes:
-      - /dev/video0:/dev/video0
+      - ${DEVICE}:/dev/video0
+      - ${WEI_DATA_DIR}:/home/app/.wei
+      - ./src:/home/app/webcam_module/src
+      - ./tests:/home/app/webcam_module/tests
     ports:
-      - 2001:2000
+      - 2000:2000
+
+  #####################
+  # WEI Core Services #
+  #####################
+  wei_server:
+    image: ghcr.io/ad-sdl/wei
+    container_name: wei_server
+    ports:
+      - 8000:8000
+    volumes:
+      - ${WORKCELLS_DIR}:/workcell_defs
+      - ${WEI_DATA_DIR}:/home/app/.wei
+      - diaspora_config:/home/app/.diaspora
+    env_file: .env
+    command: python3 -m wei.server --workcell /workcell_defs/${WORKCELL_FILENAME} --use_diaspora ${USE_DIASPORA}
+    depends_on:
+      - wei_redis
+  wei_engine:
+    image: ghcr.io/ad-sdl/wei
+    container_name: wei_engine
+    volumes:
+      - ${WORKCELLS_DIR}:/workcell_defs
+      - ${WEI_DATA_DIR}:/home/app/.wei
+    env_file: .env
+    command: python3 -m wei.engine --workcell /workcell_defs/${WORKCELL_FILENAME} --use_diaspora ${USE_DIASPORA}
+    depends_on:
+      - wei_redis
+      - wei_server
+  wei_redis:
+    image: redis
+    container_name: wei_redis
+    ports:
+      - 6379:6379
+    volumes:
+      - ${REDIS_DIR}:/data
+    command: redis-server --save 60 1 --loglevel warning
+
+################
+# Data Storage #
+################
+volumes:
+  diaspora_config:
+    driver: local

make/boilerplate.mk

@@ -0,0 +1,28 @@
+######################################
+# Boilerplate Makefile Configuration #
+# You can probably leave as is       #
+######################################
+
+# List all rules in the project
+RULES := $(foreach makefile,$(MAKEFILE_LIST),\
+	$(shell grep -E '^[a-zA-Z0-9_-]+:.*' $(makefile) | sed -E 's/^([a-zA-Z0-9_-]+):.*$$/\1/' | sort -u))
+
+# Variables that we don't save to the .env file
+ENV_FILTER := $(.VARIABLES)
+ENV_FILTER += TEMP .SHELLSTATUS PHONY_RULES
+
+#####################
+# Boilerplate Rules #
+#####################
+
+help: # Show help for each target
+	@echo ""
+	@echo "Usage: make <rule>, where <rule> is one of:"
+	@echo ""
+	@for file in $(MAKEFILE_LIST); \
+		do grep -E '^[a-zA-Z0-9 -_]+:.*#' $$file | sort | \
+			while read -r l; \
+				do printf "\033[1;32m$$(echo $$l | cut -f 1 -d':')\033[00m:$$(echo $$l | cut -f 2- -d'#')\n"; \
+			done; \
+		done \
+	| sort -u

make/config.mk

@@ -0,0 +1,67 @@
+#####################
+# WEI Configuration	#
+#####################
+
+# Adjust these parameters to meet your needs
+# You can override these at run time by running `make <target> <VARIABLE>="..."`
+
+# Project Configuration
+MODULE_NAME := webcam_module
+PROJECT_DIR := $(abspath $(MAKEFILE_DIR))
+WORKCELLS_DIR := $(PROJECT_DIR)/tests/workcell_defs
+WORKCELL_FILENAME := test_workcell.yaml
+DEVICE := /dev/video0
+
+# Python Configuration
+PYPROJECT_TOML := $(PROJECT_DIR)/pyproject.toml
+PROJECT_VERSION := $(shell grep -oP '(?<=version = ")[^"]+' $(PYPROJECT_TOML) | head -n 1)
+
+# Docker Configuration
+COMPOSE_FILE := $(PROJECT_DIR)/compose.yaml
+DOCKERFILE := $(PROJECT_DIR)/Dockerfile
+# Make sure ENV_FILE is in .gitignore or equivalent
+ENV_FILE := $(PROJECT_DIR)/.env
+REGISTRY := ghcr.io
+ORGANIZATION := ad-sdl
+IMAGE_NAME := $(MODULE_NAME)
+IMAGE := $(REGISTRY)/$(ORGANIZATION)/$(IMAGE_NAME)
+
+# APP_NAME needs to match the name of the module's service in the compose file
+APP_NAME := $(MODULE_NAME)
+# This is where the data from the workcell and your application will be stored
+# If these directories don't exist, they will be created
+WEI_DATA_DIR := $(PROJECT_DIR)/.wei
+REDIS_DIR := $(WEI_DATA_DIR)/redis
+# Whether or not to send events to Diaspora (set to true to turn on)
+USE_DIASPORA := false
+# This is the default target to run when you run `make` with no arguments
+.DEFAULT_GOAL := help
+
+########################
+# Config-related Rules #
+########################
+
+init: .env $(WEI_DATA_DIR) $(REDIS_DIR) # Do the initial configuration of the project
+
+# Generate our .env whenever we change our config
+# If you depend on files besides the makefiles to generate config,
+# add them as dependencies to the .env rule
+NOT_PHONY += .env
+.env: $(MAKEFILE_LIST)
+	@echo Generating .env...
+	@echo "# THIS FILE IS AUTOGENERATED, CHANGE THE VALUES IN THE MAKEFILE" > $(ENV_FILE)
+	@echo "USER_ID=$(shell id -u)" >> $(ENV_FILE)
+	@echo "GROUP_ID=$(shell id -g)" >> $(ENV_FILE)
+	@echo "MHF_HOST_UID=$(shell id -u)" >> $(ENV_FILE)
+	@echo "MHF_HOST_GID=$(shell id -g)" >> $(ENV_FILE)
+# The following adds every variable in the Makefiles to the .env file,
+# except for everything in ENV_FILTER and ENV_FILTER itself
+	@$(foreach v,\
+		$(filter-out $(ENV_FILTER) ENV_FILTER,$(.VARIABLES)),\
+		echo "$(v)=$($(v))" >> $(ENV_FILE);)
+
+$(WEI_DATA_DIR):
+	mkdir -p $(WEI_DATA_DIR)
+
+$(REDIS_DIR):
+	mkdir -p $(REDIS_DIR)

make/docker.mk

@@ -0,0 +1,42 @@
+############################################
+# Docker- and Docker Compose-related rules #
+############################################
+
+exec: init # Opens a shell in the APP_NAME container
+	docker compose -f $(COMPOSE_FILE) exec -u app $(APP_NAME) /bin/bash $(args)
+
+build: init # Builds the docker image for APP_NAME
+	docker build -f $(DOCKERFILE) \
+		-t ${IMAGE}:${PROJECT_VERSION} \
+		-t ${IMAGE}:latest \
+		-t ${IMAGE}:dev \
+		$(PROJECT_DIR)
+	docker compose -f $(COMPOSE_FILE) build $(args)
+
+start: init # Starts all the docker containers and detaches, allowing you to run other commands
+start: $(if $(findstring $(USE_DIASPORA),true), register_diaspora)
+	docker compose -f $(COMPOSE_FILE) up -d $(args)
+
+up: init # Starts all the docker containers and attaches, allowing you to see the logs
+	docker compose -f $(COMPOSE_FILE) up $(args)
+
+ps: init # Shows the status of all the docker containers
+	docker compose -f $(COMPOSE_FILE) ps $(args)
+
+restart: init # Restarts all the docker containers
+	docker compose -f $(COMPOSE_FILE) restart $(args)
+
+pull: init # Pull the latest versions of the required containers
+	docker compose -f ${COMPOSE_FILE} pull
+
+update: init pull build stop start # Pulls, builds, stops, and then starts all containers
+
+down: stop # Stops all the docker containers
+stop: init # Stops all the docker containers
+	docker compose -f $(COMPOSE_FILE) down $(args)
+
+logs: init # Shows the logs for all the docker containers
+	docker compose -f $(COMPOSE_FILE) logs -f $(args)
+
+remove: init # Removes all the docker containers, but preserves volumes
+	docker compose -f $(COMPOSE_FILE) down --rmi all $(args)