From a003554383d9d89fe4460b05228c03197b84be85 Mon Sep 17 00:00:00 2001 From: Marcus Lagergren <1062473+lagergren@users.noreply.github.com> Date: Thu, 14 Mar 2024 12:33:46 +0100 Subject: [PATCH] 'dev style' container works, except for reverse proxy setup, when avoiding the globally configured DNS mapping for the xqiz.it domain, which resolves to 127.0.0.1 which causes some trouble to reroute stuff. For the use case 'docker run --pull' from the Internet, we would ideally like to have the release flow for the XDK finished, and it also has a higher priority, so temporarily pausing to do that. --- .env | 13 ++++ .gitignore | 6 ++ README.md | 2 +- build.gradle.kts | 81 ++++++++++++------------ docker-compose.yaml | 38 +++++++++--- docker/.env | 4 -- docker/.env.local | 4 -- docker/Dockerfile.platform | 35 +++-------- docker/Dockerfile.platform.dev | 7 +++ docker/cert/README.md | 3 + docker/config/cfg.json | 5 ++ docker/entrypoint.sh | 50 --------------- docker/scripts/entrypoint.sh | 85 ++++++++++++++++++++++++++ docker/{ => scripts}/platform-build.sh | 0 docker/{ => scripts}/platform-down.sh | 0 docker/{ => scripts}/platform-up.sh | 0 16 files changed, 203 insertions(+), 130 deletions(-) create mode 100644 .env delete mode 100644 docker/.env delete mode 100644 docker/.env.local create mode 100644 docker/Dockerfile.platform.dev create mode 100644 docker/cert/README.md create mode 100644 docker/config/cfg.json delete mode 100755 docker/entrypoint.sh create mode 100755 docker/scripts/entrypoint.sh rename docker/{ => scripts}/platform-build.sh (100%) mode change 100644 => 100755 rename docker/{ => scripts}/platform-down.sh (100%) rename docker/{ => scripts}/platform-up.sh (100%) diff --git a/.env b/.env new file mode 100644 index 0000000..35750e4 --- /dev/null +++ b/.env @@ -0,0 +1,13 @@ +# +# This env file is not the same as the one in docker compose env_file sections. +# This sets a default enviromnent from the build, e.g. with build args. +# The others set environment variables to be used in the container. +# +COMPOSE_PROJECT_NAME=platform + +GITHUB_BRANCH_PLATFORM=${GITHUB_BRANCH:-platform-xtcplugin} + +PLATFORM_HOSTNAME=${PLATFORM_HOSTNAME:-xtc-platform.container.xqiz.it} +PLATFORM_HOSTNAME_DEV=${PLATFORM_HOSTNAME_DEV:-xtc-platform.container-dev.xqiz.it} + +XTC_VERSION=0.4.3 diff --git a/.gitignore b/.gitignore index 6096497..3d8bc08 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,9 @@ prj/ # Gradle caches # .gradle + +# +# Local environment with secrets for Docker compose (or any other system specific thing that +# should not go into source control) +# +.env.local diff --git a/README.md b/README.md index c15b2df..79bb62e 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ as a container/Dockerfile/docker in the near future, so that you won't have to d 4. Make sure you can ping the local platform address: ``` - ping xtc-platform2.localhost.xqiz.it + ping xtc-platform.localhost.xqiz.it ``` The domain name `xtc-platform.localhost.xqiz.it` should resolve to `127.0.0.1`. This allows the same xqiz.it diff --git a/build.gradle.kts b/build.gradle.kts index e52ac41..11afff1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -31,8 +31,48 @@ dependencies { xtcModule(project(":platformUI")) // runtime path } -// TODO: Automatically run a pre-pass to verify that the XTC init script is in place (until we are -// on Maven Central (and have a new XDK release from "master"that supports the XTC Plugin). +/** + * Gather all compiled XTC modules from subprojects into a single location: $rootProject/build/platform. + */ +val commonXtcOutputDir = layout.buildDirectory.dir("platform") + +allprojects { + tasks.withType().configureEach { + //outputs.dir(commonXtcOutputDir) + doLast { + copy { + val compilerOutput = outputs.files.asFileTree + compilerOutput.forEach { + logger.lifecycle("XTC module output: ${it.absolutePath} -> ${commonXtcOutputDir.get().asFile.absolutePath}") + } + from(compilerOutput) + into(commonXtcOutputDir) + } + } + } +} + +/** + * This is the run configuration, which configures all xtcRun taks for the main source set. (runXtc, runAllXtc) + * The DSL for modules to run is a list of "module { }" elements or a list of moduleName("...") statements. + * To look at the DSL for all parts of the XTC build, you can use your IDE and browse the implementing + * classes. For example, there should be a hint in IntelliJ with the type for the xtcRun element and + * the modules element (DefaultXtcRuntimeExtension and XtcRuntimeExtension.XtcRunModule, respectively). + * It is a good way to understand how the build DSL works, so you can add your own powerful XTC build + * syntax constructs and nice syntactic sugar/shorthand for things you feel should be simpler to write. + */ +xtcRun { + debug = false // Set to true to get the launcher to pause and wait for a debugger to attach. + verbose = true + stdin = System.`in` // Prevent Gradle from eating stdin; make it interactive with the Gradle process that executes the kernel. + module { + moduleName = "kernel" + moduleArg(passwordProvider) + //findProperty("keystorePassword")?.also { + // moduleArg(it.toString()) + //} + } +} /** * Lazy password resolution provider. @@ -66,43 +106,6 @@ internal val passwordProvider: Provider = provider { findProperty("keystorePassword")?.toString() ?: "" } -/** - * Gather all compiled XTC modules from subprojects into a single location: $rootProject/build/platform. - */ -val commonXtcOutputDir = layout.buildDirectory.dir("platform") - -allprojects { - tasks.withType().configureEach { - doLast { - copy { - from(outputs.files.asFileTree) - into(commonXtcOutputDir) - } - } - } -} - -/** - * This is the run configuration, which configures all xtcRun taks for the main source set. (runXtc, runAllXtc) - * The DSL for modules to run is a list of "module { }" elements or a list of moduleName("...") statements. - * To look at the DSL for all parts of the XTC build, you can use your IDE and browse the implementing - * classes. For example, there should be a hint in IntelliJ with the type for the xtcRun element and - * the modules element (DefaultXtcRuntimeExtension and XtcRuntimeExtension.XtcRunModule, respectively). - * It is a good way to understand how the build DSL works, so you can add your own powerful XTC build - * syntax constructs and nice syntactic sugar/shorthand for things you feel should be simpler to write. - */ -xtcRun { - verbose = true - //fork = false - stdin = System.`in` // Prevent Gradle from eating stdin; make it interactive with the Gradle process that executes the kernel. - module { - moduleName = "kernel" - findProperty("keystorePassword")?.also { - moduleArg(it.toString()) - } - } -} - /** * Run the XTC Platform. Note that this is a Gradle job, and as such gets is dependencies from the module path * in the Gradle plugin for all source in the project. It will use a module path precisely including diff --git a/docker-compose.yaml b/docker-compose.yaml index 0e836f4..679950c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -29,21 +29,45 @@ services: # TODO: Create a platform that is more "dev container" style, and platform: image: ghcr.io/xtclang/xdk-platform:latest + env_file: + - .env.local build: - context: docker - dockerfile: Dockerfile.platform + context: . + dockerfile: docker/Dockerfile.platform args: - DOCKER_BUILDKIT: 1 PLATFORM_PASSWORD: ${PLATFORM_PASSWORD:-password} - GITHUB_BRANCH_PLATFORM: ${GITHUB_BRANCH_PLATFORM:-platform-xtcplugin} + hostname: ${PLATFORM_HOSTNAME:-xtc-platform.local.xqiz.it} + ports: + - "8080:8080" + - "8090:8090" + + # + # Container that grabs, updates and persists source code for 'platform' + # from GitHub, and enabled cached builds in a container. + # + # A third option would be to start a dev container where the root directory + # of the platform project just is symlinked, so you can build and test yourself + # in a normal environment but get deployed elsewhere. + # + dev: + image: ghcr.io/xtclang/xdk-platform-dev:latest + depends_on: + - platform + build: + context: . + dockerfile: docker/Dockerfile.platform.dev + args: + #PLATFORM_PASSWORD: ${PLATFORM_PASSWORD:-password} + GITHUB_BRANCH_PLATFORM: $GITHUB_BRANCH_PLATFORM + environment: + DEV_CONTAINER: 1 secrets: - gradle_properties volumes: - persistent:/persistent env_file: - - docker/.env - - docker/.env.local - hostname: 'xtc-platform2.localhost.xqiz.it' + - .env.local + hostname: ${PLATFORM_HOSTNAME:-xtc-platform.local-dev.xqiz.it} #extra_hosts: # - "xtc-platform.localhost.xqiz.it:127.0.0.1" # - "xtc-platform.xqiz.it:127.0.0.10" diff --git a/docker/.env b/docker/.env deleted file mode 100644 index fd1bd5c..0000000 --- a/docker/.env +++ /dev/null @@ -1,4 +0,0 @@ -COMPOSE_PROJECT_NAME=platform - -GITHUB_BRANCH=${GITHUB_BRANCH:-master} -XTC_VERSION=0.4.3 diff --git a/docker/.env.local b/docker/.env.local deleted file mode 100644 index a899be7..0000000 --- a/docker/.env.local +++ /dev/null @@ -1,4 +0,0 @@ - -# TODO: This should be exclude from source control. - -PLATFORM_PASSWORD=${PLATFORM_PASSWORD:-password} diff --git a/docker/Dockerfile.platform b/docker/Dockerfile.platform index 17129a6..9f7ed71 100644 --- a/docker/Dockerfile.platform +++ b/docker/Dockerfile.platform @@ -1,8 +1,4 @@ -#FROM openjdk:21 -FROM ubuntu:24.04 - -ARG DOCKER_BUILDKIT=$DOCKER_BUILDKIT -ENV DOCKER_BUILD_KIT=$DOCKER_BUILDKIT +FROM openjdk:21-jdk-slim-bookworm ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 @@ -10,30 +6,16 @@ ENV DEBIAN_FRONTEND=noninteractive ARG XTC_USER=xtc ARG XTC_USER_HOME=/home/$XTC_USER -ARG NPM_SAFE_VERSION='npm@10.4.0' -ARG GITHUB_BRANCH_XVM -ARG GITHUB_BRANCH_PLATFORM ENV XTC_USER=$XTC_USER ENV XTC_USER_HOME=$XTC_USER_HOME ENV XQIZIT_HOME=$XTC_USER_HOME/xqiz.it ENV PLATFORM_HOME=$XQIZIT_HOME/platform -ENV GITHUB_BRANCH_XVM=$GITHUB_BRANCH_XVM -ENV GITHUB_BRANCH_PLATFORM=$GITHUB_BRANCH_PLATFORM - -USER root RUN apt-get update && apt-get install --no-install-recommends -y \ - iputils-ping jq sudo wget curl git openjdk-21-jdk - -# Actually, we don't need a node installation. The Gradle plugin handles everything. -#RUN curl --silent --location https://deb.nodesource.com/setup_21.x | sudo bash - -#RUN apt-get -y --no-install-recommends install nodejs && npm -g install npm@${NPM_SAFE_VERSION} # && npm -g install yarn + iputils-ping jq sudo wget curl git emacs-nox -# Install dev tools. We should separate these out later. -RUN apt-get install --no-install-recommends -y emacs-nox - -COPY *.sh /usr/local/bin +COPY docker/scripts/*.sh /usr/local/bin RUN useradd -ms /bin/bash $XTC_USER \ && passwd -d $XTC_USER \ @@ -44,11 +26,14 @@ RUN useradd -ms /bin/bash $XTC_USER \ USER $XTC_USER -# From README.md: Create xqiz.it subdirectory and config -RUN mkdir -p $PLATFORM_HOME && mkdir -p $XQIZIT_HOME/config +RUN mkdir -p $PLATFORM_HOME +RUN mkdir -p $XTC_USER_HOME/lib/xdk + +# Copy the local build (todo could also bind mount it) +COPY build/platform/* $XTC_USER_HOME/lib +COPY build/xtc/xdk/lib/* $XTC_USER_HOME/lib/xdk -# From README.md: Create port forwarding config. -COPY config/port-forwarding.conf $PLATFORM_HOME +# We should probably use a real XDK, # From README.md create: a self-signed certificate for the platform web server. For example: ARG PLATFORM_PASSWORD diff --git a/docker/Dockerfile.platform.dev b/docker/Dockerfile.platform.dev new file mode 100644 index 0000000..a111543 --- /dev/null +++ b/docker/Dockerfile.platform.dev @@ -0,0 +1,7 @@ +FROM ghcr.io/xtclang/xdk-platform:latest as platform + +ARG GITHUB_BRANCH_PLATFORM +ENV GITHUB_BRANCH_PLATFORM=$GITHUB_BRANCH_PLATFORM + +USER root + diff --git a/docker/cert/README.md b/docker/cert/README.md new file mode 100644 index 0000000..8a8df26 --- /dev/null +++ b/docker/cert/README.md @@ -0,0 +1,3 @@ +This directory contains prebuild self signed certificates for the XTC Platform hostname, and with the +key store password 'password'. They should not be used in production, naturally. The dev container +generates these values itself. \ No newline at end of file diff --git a/docker/config/cfg.json b/docker/config/cfg.json new file mode 100644 index 0000000..b4bbb37 --- /dev/null +++ b/docker/config/cfg.json @@ -0,0 +1,5 @@ +{ + "hostName": "xtc-platform.container.xqiz.it", + "httpPort": 8080, + "httpsPort": 8090 +} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index cebf1e3..0000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# This is more of a devcontainer than a production container, since we set up a build enviromnment and -# make it possible to update and build sources from it. This should be split into separate responsibiliteis -# as -echo "Entrypoint for Platform..." - -function check_name_resolution() { - ping -c 1 xtc-platform.localhost.xqiz.it - if [ $? != 0 ]; then - echo "Ping to localhost failed using xtc-platform.localhost.xqiz.it" - exit 1 - fi - echo "xtc-platform.localhost.xqiz.it resolves and responds to ping." -} - -# Under root build we have have src and .gradle - -export PLATFORM_DIR=$HOME/build - -# Ensure persistent docker volume is set up, and link our secrets and build and Gradle cache dirs from it. -sudo chown -R $XTC_USER:$XTC_USER /persistent -ln -s /persistent $PLATFORM_DIR -export GRADLE_USER_HOME=$HOME/.gradle -if [ ! -d $PLATFORM_DIR/gradle ]; then - echo "Creating Gradle user home: $GRADLE_USER_HOME" - mkdir -p $PLATFORM_DIR/gradle -fi -ln -s $PLATFORM_DIR/gradle $GRADLE_USER_HOME -if [ ! -e $GRADLE_USER_HOME/gradle.properties ]; then - echo "Linking gradle.properties" - ln -s /var/run/secrets/gradle_properties $GRADLE_USER_HOME/gradle.properties -fi - -source /usr/local/bin/platform-build.sh - -check_name_resolution -check_updated_source -check_platform_build - -pushd $SRC_DIR -./gradlew run - -# Pass any remaining args or CMD on to the run command. -if [ -z "${@}" ]; then - echo "No extra entrypoint arguments. Container exiting from $0." -else - echo "Handing over entrypoint arguments to exec: ${@}" - exec "${@}" -fi diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh new file mode 100755 index 0000000..0e180f0 --- /dev/null +++ b/docker/scripts/entrypoint.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +# This is more of a devcontainer than a production container, since we set up a build enviromnment and +# make it possible to update and build sources from it. This should be split into separate responsibiliteis +# as +echo "Entrypoint for Platform..." + +export PLATFORM_DIR=$HOME/build + +function check_file() { + if [ -z "$1" ]; then + echo "No file name provided to check." + exit 1 + fi + if [ ! -f "$1" ]; then + echo "File $1 not found." + exit 1 + fi + _len=$(state --printf "%s" "$1") + if [ $_len -le 0 ]; then + echo "File "$1" reports size as <= bytes." + exit 1 + fi +} + +function check_name_resolution() { + ping -c 1 xtc-platform.localhost.xqiz.it + if [ $? != 0 ]; then + echo "Ping to localhost failed using xtc-platform.localhost.xqiz.it" + exit 1 + fi + echo "xtc-platform.localhost.xqiz.it resolves and responds to ping." +} + +# Ensure persistent docker volume is set up, and link our secrets and build and Gradle cache dirs from it. +function ensure_volume() { + sudo chown -R $XTC_USER:$XTC_USER /persistent + ln -s /persistent $PLATFORM_DIR + export GRADLE_USER_HOME=$HOME/.gradle + if [ ! -d $PLATFORM_DIR/gradle ]; then + echo "Creating Gradle user home: $GRADLE_USER_HOME" + mkdir -p $PLATFORM_DIR/gradle + fi + ln -s $PLATFORM_DIR/gradle $GRADLE_USER_HOME + if [ ! -e $GRADLE_USER_HOME/gradle.properties ]; then + echo "Linking gradle.properties" + ln -s /var/run/secrets/gradle_properties $GRADLE_USER_HOME/gradle.properties + fi +} + +check_name_resolution + +if [ -n "$DEV_CONTAINER" ]; then + echo "Dev container detected - syncing out source." + ensure_volumes + source /usr/local/bin/platform-build.sh + check_updated_source + check_platform_build + pushd $SRC_DIR + ./gradlew run + popd +else + echo "Prod container detected - everything should be installed already." + echo "Verifying installation." + pushd $HOME/lib + check_file "common.xtc" + check_file "host.xtc" + check_file "kernel.xtc" + check_file "platformDB.xtc" + check_file "platformUI.xtc" + check_file "xdk/javatools.jar" + popd + + export "Setting up aliases." + alias xcc="java -jar $HOME/lib/xdk/javatools.jar xcc" + alias xec"=java -jar $HOME/lib/xdk/ +fi + +# Pass any remaining args or CMD on to the run command. +if [ -z "${@}" ]; then + echo "No extra entrypoint arguments. Container exiting from $0." +else + echo "Handing over entrypoint arguments to exec: ${@}" + exec "${@}" +fi diff --git a/docker/platform-build.sh b/docker/scripts/platform-build.sh old mode 100644 new mode 100755 similarity index 100% rename from docker/platform-build.sh rename to docker/scripts/platform-build.sh diff --git a/docker/platform-down.sh b/docker/scripts/platform-down.sh similarity index 100% rename from docker/platform-down.sh rename to docker/scripts/platform-down.sh diff --git a/docker/platform-up.sh b/docker/scripts/platform-up.sh similarity index 100% rename from docker/platform-up.sh rename to docker/scripts/platform-up.sh