[WIP] Preloading with custom Jupyter notebook

.gitignore

@@ -82,3 +82,4 @@ secrets.yaml
 # faker
 official.csv
 fact.png
+jupyterhub/.env-dev

README.md

@@ -36,7 +36,9 @@ There is a few other documentation links available on the [website](https://clow
 ## Installation
 
 The easiest way of running Clowder v2 is checking out the [code](https://github.com/clowder-framework/clowder2)
-and running `docker compose up` in the main directory.
+and running `docker compose up` in the main directory. If you would like to run Clowder with JupyterHub,
+you can use our script `docker-prod.sh` to start the services. Run `./docker-prod.sh prod up` to start the services
+and `./docker-prod.sh prod down` to stop them.
 
 Helm charts are available for running Clowder v2 on Kubernetes. See the [helm](https://github.com/clowder-framework/clowder2/tree/main/deployments/kubernetes/charts) directory for more information.
 

docker-compose.dev.yml

@@ -67,6 +67,8 @@ services:
 
   postgres:
       image: postgres
+      networks:
+        - clowder2
       volumes:
         - postgres_data:/var/lib/postgresql/data
       environment:
@@ -79,6 +81,8 @@ services:
       volumes:
         - ./scripts/keycloak/clowder-realm-dev.json:/opt/keycloak/data/import/realm.json:ro
         - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro
+      networks:
+        - clowder2
       command:
         - start-dev
         - --http-relative-path /keycloak
@@ -172,6 +176,7 @@ services:
 
 networks:
   clowder2:
+    name: clowder2
 
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.

docker-compose.jupyter-dev.yml

@@ -0,0 +1,30 @@
+version: '3'
+services:
+  jupyterhub:
+    build:
+        context: jupyterhub
+        dockerfile: Dockerfile.jupyterhub
+        args:
+              JUPYTERHUB_VERSION: 4
+    restart: always
+    networks:
+      - clowder2
+    volumes:
+      # The JupyterHub configuration file
+      - ./jupyterhub/jupyterhub_dev_config.py:/srv/jupyterhub/jupyterhub_config.py:ro
+      # Bind Docker socket on the hostso we can connect to the daemon from
+      # within the container
+      - /var/run/docker.sock:/var/run/docker.sock:rw
+      # Bind Docker volume on host for JupyterHub database and cookie secrets
+      - jupyterhub-data:/data
+    ports:
+      - "8765:8000"
+    env_file:
+        -   jupyterhub/.env-dev
+    command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py
+
+    depends_on:
+      - keycloak
+
+volumes:
+    jupyterhub-data:

docker-compose.jupyter.yml

@@ -0,0 +1,33 @@
+version: '3'
+services:
+  jupyterhub:
+    build:
+        context: jupyterhub
+        dockerfile: Dockerfile.jupyterhub
+        args:
+              JUPYTERHUB_VERSION: 4
+    restart: always
+    networks:
+      - clowder2
+    volumes:
+      # The JupyterHub configuration file
+      - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro
+      # Bind Docker socket on the hostso we can connect to the daemon from
+      # within the container
+      - /var/run/docker.sock:/var/run/docker.sock:rw
+      # Bind Docker volume on host for JupyterHub database and cookie secrets
+      - jupyterhub_data:/data
+    env_file:
+        -   jupyterhub/.env
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.jupyterhub.rule=PathPrefix(`/jupyterhub`)"
+      - "traefik.http.services.jupyterhub.loadbalancer.server.port=8000"
+
+    command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py
+
+    depends_on:
+      - keycloak
+
+volumes:
+    jupyterhub_data:

docker-dev.sh

@@ -7,3 +7,12 @@ if [ "$1" = "down" ]
 then
   docker-compose -f docker-compose.dev.yml -p clowder2-dev down
 fi
+if [ "$1" = "jupyter" ] && [ "$2" = "up" ]
+then
+  docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev  up -d --build
+fi
+
+if [ "$1" = "jupyter" ] && [ "$2" = "down" ]
+then
+  docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev  down
+fi

docker-prod.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+if [ "$1" = "prod" ] && [ "$2" = "up" ]
+then
+  docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml up -d
+fi
+
+if [ "$1" = "prod" ] && [ "$2" = "down" ]
+then
+  docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml down
+fi

docs/docs/devs/getstarted.md

@@ -47,6 +47,8 @@ section below).
 - Running `docker-compose logs -f` displays the live logs for all containers. To view the logs of individual containers,
   provide the container name. For example, for viewing the backend module logs, run `docker-compose logs -f backend`.
 - Running `./docker-dev.sh down` brings down the required services.
+- If you want to run the jupyterhub, you can run `./docker-dev.sh jupyter up`. The jupyterhub will be available at
+  `http://localhost:8765`. You can bring it down using `./docker-dev.sh jupyter down`.
 
 **Note:** `./docker-dev.sh` sets the project name flag to `-p clowder2-dev`. This is so that the dev containers
 don't get mixed with the production containers if the user is running both on the same machine using `docker-compose.yml`.

frontend/package.json

@@ -12,7 +12,7 @@
 		"start-message": "babel-node tools/startMessage.js",
 		"prestart": "npm-run-all --parallel start-message",
 		"start": "npm-run-all --parallel open:src",
-		"start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && npm run start",
+		"start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && export JUPYTERHUB_URL=http://localhost:8765 && npm run start",
 		"open:src": "babel-node tools/srcServer.js",
 		"open:dist": "babel-node tools/distServer.js",
 		"lint:watch": "npm run lint --watch",

frontend/src/app.config.ts

@@ -9,6 +9,7 @@ interface Config {
 	hostname: string;
 	apikey: string;
 	GHIssueBaseURL: string;
+	jupyterHubURL: string;
 	KeycloakBaseURL: string;
 	KeycloakLogin: string;
 	KeycloakLogout: string;
@@ -65,6 +66,10 @@ config["KeycloakRegister"] = `${config.KeycloakBaseURL}/register`;
 config["searchEndpoint"] = `${hostname}/api/v2/elasticsearch`;
 config["publicSearchEndpoint"] = `${hostname}/api/v2/public_elasticsearch`;
 
+// jupterhub
+const localJupyterhubURL: string = `${config.hostname}/jupyterhub`;
+config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL;
+
 // refresh token time interval
 config["refreshTokenInterval"] = 1000 * 60; // 1 minute
 // updated extractor logs

frontend/src/components/Layout.tsx

@@ -23,6 +23,7 @@ import { RootState } from "../types/data";
 import { AddBox, Explore } from "@material-ui/icons";
 import HistoryIcon from "@mui/icons-material/History";
 import GroupIcon from "@mui/icons-material/Group";
+import MenuBookIcon from "@mui/icons-material/MenuBook";
 import Gravatar from "react-gravatar";
 import PersonIcon from "@mui/icons-material/Person";
 import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined";
@@ -41,6 +42,8 @@ import AdminPanelSettingsIcon from "@mui/icons-material/AdminPanelSettings";
 import { Footer } from "./navigation/Footer";
 import BuildIcon from "@mui/icons-material/Build";
 
+import config from "../app.config";
+
 const drawerWidth = 240;
 
 const Main = styled("main", { shouldForwardProp: (prop) => prop !== "open" })<{
@@ -439,6 +442,22 @@ export default function PersistentDrawerLeft(props) {
 					</ListItem>
 				</List>
 				<Divider />
+				{/*TODO: Need to make link dynamic */}
+				<List>
+					<ListItem key={"jupyter"} disablePadding>
+						<ListItemButton
+							component={RouterLink}
+							to={config.jupyterHubURL}
+							target="_blank"
+							rel="noopener noreferrer"
+						>
+							<ListItemIcon>
+								<MenuBookIcon />
+							</ListItemIcon>
+							<ListItemText primary={"Jupyter Notebook"} />
+						</ListItemButton>
+					</ListItem>
+				</List>
 				<List>
 					<ListItem key={"feeds"} disablePadding>
 						<ListItemButton component={RouterLink} to="/feeds">

frontend/webpack.config.dev.js

@@ -8,6 +8,9 @@ import ESLintPlugin from "eslint-webpack-plugin";
 console.log(
 	`the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}`
 );
+console.log(
+	`the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}`
+)
 
 export default {
 	mode: "development",
@@ -40,6 +43,7 @@ export default {
 				CLOWDER_REMOTE_HOSTNAME: JSON.stringify(
 					process.env.CLOWDER_REMOTE_HOSTNAME
 				),
+				JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL),
 				APIKEY: JSON.stringify(process.env.APIKEY),
 				KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL),
 			},

frontend/webpack.config.prod.js

@@ -11,6 +11,9 @@ import TerserPlugin from "terser-webpack-plugin";
 console.log(
 	`the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}`
 );
+console.log(
+	`the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}`
+)
 
 export default {
 	mode: "production",
@@ -47,6 +50,7 @@ export default {
 				CLOWDER_REMOTE_HOSTNAME: JSON.stringify(
 					process.env.CLOWDER_REMOTE_HOSTNAME
 				),
+				JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL),
 				APIKEY: JSON.stringify(process.env.APIKEY),
 				KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL),
 			},

jupyterhub/.env-example

@@ -0,0 +1,14 @@
+# Example configuration file for Clowder JupyterHub
+KEYCLOAK_HOSTNAME="keycloak:8080/keycloak"
+# Development mode use the following line instead
+#KEYCLOAK_HOSTNAME="keycloak:8080/keycloak"
+KEYCLOAK_AUDIENCE="clowder"
+KEYCLOAK_REALM="clowder"
+JUPYTERHUB_ADMIN="admin"
+#Change network name to the one created by docker-compose
+DOCKER_NETWORK_NAME="clowder2_clowder2"
+DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest"
+DOCKER_NOTEBOOK_DIR="/home/jovyan/work"
+JUPYTERHUB_CRYPT_KEY=""
+CLOWDER_URL="localhost"
+PROD_DEPLOYMENT="false"