Merge branch 'pwncollege:master' into master

.github/workflows/test.yml

@@ -1,16 +1,21 @@
 name: Test building and running dojo
-on: [push, pull_request]
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '42 06 * * *'
 jobs:
   smoketest:
     runs-on: ubuntu-22.04
-    timeout-minutes: 10
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v3
       - uses: docker/setup-buildx-action@v3
       - uses: docker/build-push-action@v5
         with:
           tags: dojo-test
           load: true
+          no-cache: ${{ github.event_name == 'schedule' }}
           # cache-from: type=gha
           # cache-to: type=gha,mode=max
 

.gitignore

@@ -1,3 +1,5 @@
 data/
 *.pyc
 .DS_Store
+opt/
+sensai/

Dockerfile

@@ -13,14 +13,19 @@ RUN apt-get update && \
         iproute2 \
         iputils-ping \
         host \
-        htop
+        htop \
+        zfsutils-linux \
+        unzip
 
 RUN curl -fsSL https://get.docker.com | /bin/sh
 RUN echo '{ "data-root": "/opt/pwn.college/data/docker", "builder": {"Entitlements": {"security-insecure": true}} }' > /etc/docker/daemon.json
 
 # TODO: this can be removed with docker-v22 (buildx will be default)
 RUN docker buildx install
 
+# install aws cli (for cloud backups)
+RUN cd /tmp && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install && rm -rf awscliv2.zip ./aws
+
 RUN git clone --branch 3.6.0 https://github.com/CTFd/CTFd /opt/CTFd
 
 RUN wget -O /etc/docker/seccomp.json https://raw.githubusercontent.com/moby/moby/master/profiles/seccomp/default.json
@@ -31,10 +36,13 @@ RUN ln -s /opt/pwn.college/etc/systemd/system/pwn.college.backup.service /etc/sy
 RUN ln -s /opt/pwn.college/etc/systemd/system/pwn.college.backup.timer /etc/systemd/system/pwn.college.backup.timer
 RUN ln -s /opt/pwn.college/etc/systemd/system/pwn.college.cachewarmer.service /etc/systemd/system/pwn.college.cachewarmer.service
 RUN ln -s /opt/pwn.college/etc/systemd/system/pwn.college.cachewarmer.timer /etc/systemd/system/pwn.college.cachewarmer.timer
+RUN ln -s /opt/pwn.college/etc/systemd/system/pwn.college.cloud.backup.service /etc/systemd/system/pwn.college.cloud.backup.service
+RUN ln -s /opt/pwn.college/etc/systemd/system/pwn.college.cloud.backup.timer /etc/systemd/system/pwn.college.cloud.backup.timer
 RUN ln -s /etc/systemd/system/pwn.college.service /etc/systemd/system/multi-user.target.wants/pwn.college.service
 RUN ln -s /etc/systemd/system/pwn.college.logging.service /etc/systemd/system/multi-user.target.wants/pwn.college.logging.service
 RUN ln -s /etc/systemd/system/pwn.college.backup.timer /etc/systemd/system/timers.target.wants/pwn.college.backup.timer
 RUN ln -s /etc/systemd/system/pwn.college.cachewarmer.timer /etc/systemd/system/timers.target.wants/pwn.college.cachewarmer.timer
+RUN ln -s /etc/systemd/system/pwn.college.cloud.backup.timer /etc/systemd/system/timers.target.wants/pwn.college.cloud.backup.timer
 
 RUN mkdir -p /opt/pwn.college
 ADD . /opt/pwn.college

README.md

@@ -69,13 +69,43 @@ The following options are available:
 - `challenge-mini`: Adds a minified desktop (the default).
 - `challenge-full`: The full (70+ GB) setup.
 
+## Updating
+
+When updating your dojo deployment, there is only one supported method in the `dojo` directory:
+
+```sh
+docker kill pwncollege/dojo
+docker rm pwncollege/dojo
+git pull
+docker build -t pwncollege/dojo "$DOJO_PATH"
+docker run --privileged -d -v "${DOJO_PATH}:/opt/pwn.college:shared" -p 22:22 -p 80:80 -p 443:443 --name dojo pwncollege/dojo
+```
+
+This will cause downtime when the dojo is rebuilding.
+
+Some changes _can_ be applied without a complete restart, however this is not guaranteed.
+
+If you really know what you're doing (the changes that you're pulling in are just to `ctfd`), inside the `pwncollege/dojo` container you can do the following:
+
+```sh
+dojo update
+```
+
+Note that `dojo update` is not guaranteed to be successful and should only be used if you fully understand each commit/change that you are updating.
+
 ## Customization
 
 _All_ dojo data will be stored in the `./data` directory.
 
 Once logged in, you can add a dojo by visiting `/dojos/create`. Dojos are contained within git repositories. 
 Refer to [the example dojo](https://github.com/pwncollege/example-dojo) for more information.
 
+## Cloud Backups
+
+If configured properly, the dojo will store the hourly database backups into an S3 bucket of your choosing.
+
+TODO ADD MORE HERE
+
 ## Contributing
 
 We love Pull Requests! 🌟
@@ -84,5 +114,4 @@ Send a PR so everyone can benefit.
 For more substantial changes, open an issue to ensure we're on the same page.
 Together, we make this project better for all! 🚀
 
-You can run the dojo CI testcases locally using [act](https://github.com/nektos/act).
-They should run using the "medium" image.
+You can run the dojo CI testcases locally using `test/local-tester.sh`.

challenge/Dockerfile

@@ -14,8 +14,7 @@ ARG INSTALL_GECKODRIVER=${DEFAULT_INSTALL_SELECTION}
 ARG INSTALL_BURPSUITE=${DEFAULT_INSTALL_SELECTION}
 ARG INSTALL_BUSYBOX=${DEFAULT_INSTALL_SELECTION}
 ARG INSTALL_GLOW=${DEFAULT_INSTALL_SELECTION}
-ARG INSTALL_DESKTOP_BASE=${DEFAULT_INSTALL_SELECTION}
-ARG INSTALL_XFCE=${DEFAULT_INSTALL_SELECTION}
+ARG INSTALL_DESKTOP=${DEFAULT_INSTALL_SELECTION}
 ARG INSTALL_VIRTIOFSD=${DEFAULT_INSTALL_SELECTION}
 ARG INSTALL_IDA_FREE=${DEFAULT_INSTALL_SELECTION}
 ARG INSTALL_BINJA_FREE=${DEFAULT_INSTALL_SELECTION}
@@ -50,6 +49,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     apt-get clean && rm -rf /var/lib/apt/lists/*
         ca-certificates
         curl
+        netcat-openbsd
         socat
         sudo
         vim
@@ -81,6 +81,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
         autoconf
         bc
         bison
+        cargo
         clang
         cmake
         cpio
@@ -125,6 +126,22 @@ EOF
 
 ################################################################################
 
+FROM builder as builder-github-tools
+RUN <<EOF
+    git clone --depth 1 --recurse-submodules https://github.com/pwncollege/dojjail /opt/dojjail
+    pip install -e /opt/dojjail
+
+    git clone --depth 1 --recurse-submodules https://github.com/zolutal/kropr /opt/kropr
+    cd /opt/kropr
+    cargo install --path . --root /tmp
+    mv /tmp/bin/ropr /usr/bin/kropr
+    chmod +x /usr/bin/kropr
+
+    git clone --depth 1 --recurse-submodules https://github.com/zolutal/gdb-pt-dump /opt/pt-dump
+EOF
+
+################################################################################
+
 FROM builder as builder-kernel-no
 RUN mkdir /opt/linux
 FROM builder as builder-kernel-yes
@@ -203,6 +220,10 @@ RUN <<EOF
 
     git clone --depth 1 https://github.com/hugsy/gef /opt/gef
     git clone --depth 1 https://github.com/jerdna-regeiz/splitmind /opt/splitmind
+
+    git clone --depth 1 https://github.com/nccgroup/libslub /opt/libslub
+    cd /opt/libslub
+    pip install -r requirements.txt
 EOF
 
 FROM builder-gdb-${INSTALL_GDB} as builder-gdb
@@ -355,36 +376,22 @@ FROM builder-virtiofsd-${INSTALL_VIRTIOFSD} as builder-virtiofsd
 
 ################################################################################
 
-FROM essentials as builder-desktop-base-no
-FROM essentials as builder-desktop-base-yes
+FROM essentials as builder-desktop-no
+FROM essentials as builder-desktop-yes
 
 ARG UBUNTU_VERSION=20.04
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     [ "${UBUNTU_VERSION}" == "20.04" ] && TGR=common || TGR=tools; \
-    apt-get update && xargs apt-get install --no-install-recommends -yqq <<EOF && \
+    apt-get update && xargs apt-get install -yqq <<EOF && \
+    apt-get -y remove --purge at-spi2-core tumbler gvfs-* && \
+    apt-get -y autoremove && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
         novnc
         tigervnc-standalone-server
         tigervnc-$TGR
         websockify
-        fluxbox
         xterm
-EOF
-
-FROM builder-desktop-base-${INSTALL_DESKTOP_BASE} as builder-desktop-base
-
-################################################################################
-
-FROM builder-desktop-base-no as builder-desktop-xfce-no
-FROM builder-desktop-base-yes as builder-desktop-xfce-yes
-
-RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
-    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    apt-get update && xargs apt-get install -yqq <<EOF && \
-    apt-get -y remove --purge at-spi2-core tumbler gvfs-* && \
-    apt-get -y autoremove && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
         dbus-x11
         mousepad
         xclip
@@ -398,7 +405,7 @@ COPY desktop/pwncollege_background.jpg /usr/share/backgrounds/
 
 RUN rm /etc/xdg/autostart/*
 
-FROM builder-desktop-xfce-${INSTALL_XFCE} as builder-desktop-xfce
+FROM builder-desktop-${INSTALL_DESKTOP} as builder-desktop
 
 ################################################################################
 
@@ -485,6 +492,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
         expect
         fish
         findutils
+	finger
         firefox
         gcc-aarch64-linux-gnu
         gdb
@@ -504,7 +512,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
         nano
         neovim
         net-tools
-        netcat-openbsd
         nmap
         openssh-server
         parallel
@@ -562,6 +569,7 @@ RUN mkdir /opt/pwn.college
 COPY docker-initialize.sh /opt/pwn.college/docker-initialize.sh
 COPY docker-entrypoint.d /opt/pwn.college/docker-entrypoint.d
 COPY docker-entrypoint.sh /opt/pwn.college/docker-entrypoint.sh
+COPY services.d /opt/pwn.college/services.d
 COPY setuid_interpreter.c /opt/pwn.college/setuid_interpreter.c
 COPY bash.bashrc /opt/pwn.college/bash.bashrc
 COPY vm /opt/pwn.college/vm
@@ -583,8 +591,7 @@ FROM builder-tools-apt as builder-windows-no
 FROM builder-tools-apt as builder-windows-base
 
 WORKDIR /opt/windows
-COPY windows/Autounattend.xml windows/setup.ps1 windows/startup.ps1 windows/sshd_config ./
-COPY windows/challenge-proxy ./challenge-proxy
+COPY windows/Autounattend.xml windows/setup.ps1 windows/config_startup.ps1 windows/sshd_config ./
 COPY windows/shutdown.py .
 
 
@@ -602,8 +609,8 @@ RUN <<EOF
     sed -i "s/INSTALL_IDA_FREE/yes/g" ./setup.ps1
     touch ./practice-mode-enabled
     mkfs.fat -F 12 -C ./build/floppy.img 1440
-    mcopy -si ./build/floppy.img ./Autounattend.xml ./setup.ps1 ./startup.ps1 ./practice-mode-enabled ./sshd_config ./challenge-proxy/ ::
-    rm -rf ./Autounattend.xml ./setup.ps1 ./startup.ps1 ./practice-mode-enabled ./sshd_config ./challenge-proxy/
+    mcopy -si ./build/floppy.img ./Autounattend.xml ./setup.ps1 ./config_startup.ps1 ./practice-mode-enabled ./sshd_config ::
+    rm -rf ./Autounattend.xml ./setup.ps1 ./practice-mode-enabled ./sshd_config
 EOF
 
 FROM builder-windows-base as builder-windows-base-ida-no
@@ -619,11 +626,11 @@ RUN <<EOF
     rm -rf /var/lib/apt/lists/*
     touch ./practice-mode-enabled
     mkfs.fat -F 12 -C ./build/floppy.img 1440
-    mcopy -si ./build/floppy.img ./Autounattend.xml ./setup.ps1 ./startup.ps1 ./practice-mode-enabled ./sshd_config ./challenge-proxy/ ::
-    rm -rf ./Autounattend.xml ./setup.ps1 ./startup.ps1 ./practice-mode-enabled ./sshd_config ./challenge-proxy/
+    mcopy -si ./build/floppy.img ./Autounattend.xml ./setup.ps1 ./practice-mode-enabled ./sshd_config ::
+    rm -rf ./Autounattend.xml ./setup.ps1 ./practice-mode-enabled ./sshd_config
 EOF
 
-from builder-windows-base-ida-${INSTALL_IDA_FREE} as builder-windows-yes
+from builder-windows-base-ida-${INSTALL_IDA_FREE} as builder-windows-install
 
 RUN <<EOF
     echo "Building virtio drivers disk"
@@ -670,6 +677,11 @@ RUN --security=insecure <<EOF
     rm -rf ./build/monitor.sock
 EOF
 
+# This step also does the "post-install building as hacker with admin privs"
+from builder-windows-install as builder-windows-yes
+
+COPY windows/challenge-proxy.c windows/post_install.sh windows/post_install.ps1 windows/startup.ps1 ./
+
 RUN --security=insecure <<EOF
     echo "Performing initial bootup"
     qemu-system-x86_64 \
@@ -687,10 +699,8 @@ RUN --security=insecure <<EOF
       -drive "file=./build/server-2022.iso,media=cdrom" \
       -drive "file=./build/virtio-win-processed.iso,media=cdrom" \
       -drive "file=./build/clean.qcow2,if=virtio,cache=writeback,discard=ignore,format=qcow2" &
-    python3 ./shutdown.py
+    ./post_install.sh
     rm -f ./build/monitor.sock
-    touch ./build/image-built
-    echo "Windows image built"
 EOF
 
 FROM builder-windows-${INSTALL_WINDOWS} as builder-windows
@@ -724,11 +734,12 @@ EOF
 
 FROM challenge-nano as challenge-micro
 COPY --link --from=builder-code-server / /
+COPY --link --from=builder-github-tools / /
 
 ################################################################################
 
 FROM challenge-micro as challenge-mini
-COPY --link --from=builder-desktop-base / /
+COPY --link --from=builder-desktop-yes / /
 
 ################################################################################
 
@@ -748,7 +759,6 @@ COPY --link --from=builder-geckodriver / /
 COPY --link --from=builder-burpsuite / /
 COPY --link --from=builder-busybox / /
 COPY --link --from=builder-glow / /
-COPY --link --from=builder-desktop-xfce / /
 COPY --link --from=builder-virtiofsd /opt/virtiofsd /opt/virtiofsd
 COPY --link --from=builder-desktop-ida-free / /
 COPY --link --from=builder-desktop-binja-free / /
@@ -782,7 +792,6 @@ COPY --link --from=builder-geckodriver-yes / /
 COPY --link --from=builder-burpsuite-yes / /
 COPY --link --from=builder-busybox-yes / /
 COPY --link --from=builder-glow-yes / /
-COPY --link --from=builder-desktop-xfce-yes / /
 COPY --link --from=builder-virtiofsd-yes /opt/virtiofsd /opt/virtiofsd
 COPY --link --from=builder-desktop-ida-free / /
 COPY --link --from=builder-desktop-binja-free / /

challenge/docker-entrypoint.d/39_start_gui.sh → challenge/services.d/desktop

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+[ -f /tmp/.dojo/vnc/vncserver.pid ] && exit
+exec 2> /tmp/.dojo/service-desktop.log
+
 mkdir -p /tmp/.dojo/vnc /home/hacker/.vnc
 
 container_id="$(cat /.authtoken)"
@@ -39,7 +42,8 @@ start-stop-daemon --start \
                   >>/tmp/.dojo/vnc/websockify.log \
                   2>&1
 
-seq 1 50 | while read cnt; do sleep 0.1; [ -e /tmp/.X11-unix/X42 ] && break; done
+until [ -e /tmp/.X11-unix/X42 ]; do sleep 0.1; done
+until curl -s dojo-user:6081 >/dev/null; do sleep 0.1; done
 
 export DISPLAY=:42
 

challenge/docker-entrypoint.d/38_start_windows_gui.sh → challenge/services.d/desktop-windows

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+[ -f /tmp/.dojo/vnc/websockify-windows.pid ] && exit
+exec 2> /tmp/.dojo/service-desktop-windows.log
+
 mkdir -p /tmp/.dojo/vnc /home/hacker/.vnc
 start-stop-daemon --start \
                   --pidfile /tmp/.dojo/vnc/websockify-windows.pid \
@@ -14,3 +17,5 @@ start-stop-daemon --start \
                   </dev/null \
                   >>/tmp/.dojo/vnc/websockify-windows.log \
                   2>&1
+
+until curl -s dojo-user:6082 >/dev/null; do sleep 0.1; done

challenge/docker-entrypoint.d/20_start_code_server.sh → challenge/services.d/vscode

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+[ -f /tmp/.dojo/code-server/code-server.pid ] && exit
+exec 2> /tmp/.dojo/service-vscode.log
+
 mkdir -p /tmp/.dojo/code-server
 start-stop-daemon --start \
                   --pidfile /tmp/.dojo/code-server/code-server.pid \
@@ -15,3 +18,5 @@ start-stop-daemon --start \
                   </dev/null \
                   >>/tmp/.dojo/code-server/code-server.log \
                   2>&1
+
+until curl -s dojo-user:6080 >/dev/null; do sleep 0.1; done