Refactoring docker compose files for easier execution

config/docker-compose-common-services.yaml

@@ -0,0 +1,27 @@
+version: '2.4'
+
+services:
+
+  cloud-agent:
+    container_name: cloud-agent
+    image: index.docker.io/sourcegraph/src-tunnel-agent:2024-02-15-04-02-110c2ea9
+    volumes:
+      - ../config/cloud-agent-service-account-key.json:/sourcegraph/cloud-agent-service-account-key.json:ro
+      - ../config/cloud-agent-config.yaml:/sourcegraph/cloud-agent-config.yaml:ro
+    command: ["-config=/sourcegraph/cloud-agent-config.yaml"]
+    restart: always
+    networks:
+      - sourcegraph
+
+  src-serve-git:
+    # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network
+    container_name: src-serve-git-ubuntu.local
+    image:  index.docker.io/sourcegraph/src-cli:latest
+    volumes:
+      - ../src-serve-root/:/sourcegraph/src-serve-root:ro
+    command: "serve-git -addr :443 /sourcegraph/src-serve-root"
+    networks:
+      - sourcegraph
+
+networks:
+  sourcegraph:

host-ubuntu/docker-compose-override.yaml

@@ -0,0 +1,5 @@
+services:
+
+  src-serve-git:
+    # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network
+    container_name: src-serve-git-ubuntu.local

host-ubuntu/docker-compose.yaml

@@ -1,27 +1,4 @@
-version: '2.4'
-
-services:
-
-  cloud-agent:
-    container_name: cloud-agent
-    image: index.docker.io/sourcegraph/src-tunnel-agent:2024-02-05-22-28-333320ee
-    volumes:
-      - ../config/cloud-agent-service-account-key.json:/sourcegraph/cloud-agent-service-account-key.json:ro
-      - ../config/cloud-agent-config.yaml:/sourcegraph/cloud-agent-config.yaml:ro
-    command: ["-config=/sourcegraph/cloud-agent-config.yaml"]
-    restart: always
-    networks:
-      - sourcegraph
-
-  src-serve-git:
-    # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network
-    container_name: src-serve-git-ubuntu.local
-    image:  index.docker.io/sourcegraph/src-cli:latest
-    volumes:
-      - ../src-serve-root/:/sourcegraph/src-serve-root:ro
-    command: "serve-git -addr :443 /sourcegraph/src-serve-root"
-    networks:
-      - sourcegraph
-
-networks:
-  sourcegraph:
+include:
+  - path:
+    - ../config/docker-compose-common-services.yaml
+    - docker-compose-override.yaml

host-wsl2-ubuntu/docker-compose-override.yaml

@@ -0,0 +1,5 @@
+services:
+
+  src-serve-git:
+    # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network
+    container_name: src-serve-git-wsl.local

host-wsl2-ubuntu/docker-compose.yaml

@@ -1,27 +1,4 @@
-version: '2.4'
-
-services:
-
-  cloud-agent:
-    container_name: cloud-agent
-    image: index.docker.io/sourcegraph/src-tunnel-agent:2024-02-15-04-02-110c2ea9
-    volumes:
-      - ../config/cloud-agent-service-account-key.json:/sourcegraph/cloud-agent-service-account-key.json:ro
-      - ../config/cloud-agent-config.yaml:/sourcegraph/cloud-agent-config.yaml:ro
-    command: ["-config=/sourcegraph/cloud-agent-config.yaml"]
-    restart: always
-    networks:
-      - sourcegraph
-
-  src-serve-git:
-    # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network
-    container_name: src-serve-git-wsl.local
-    image:  index.docker.io/sourcegraph/src-cli:latest
-    volumes:
-      - ../src-serve-root/:/sourcegraph/src-serve-root:ro
-    command: "serve-git -addr :443 /sourcegraph/src-serve-root"
-    networks:
-      - sourcegraph
-
-networks:
-  sourcegraph:
+include:
+  - path:
+    - ../config/docker-compose-common-services.yaml
+    - docker-compose-override.yaml

host-wsl2-ubuntu/pull-build-start.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# To be used in a cronjob to always pull and build the latest
+# every 10 minutes
+# so that the running container is only 10 minutes behind the latest commit in the branch
+
+# crontab -e
+# */10 * * * * bash /sourcegraph/implementation-bridges/host-wsl2-ubuntu/pull-build-start.sh >> /sourcegraph/implementation-bridges/host-wsl2-ubuntu/pull-build-start.log 2>&1
+
+repo_build_path="/sourcegraph/implementation-bridges/repo-converter/build"
+
+git -C $repo_build_path pull
+
+docker compose -f $repo_build_path/docker-compose.yaml up -d --build

repo-converter/build/docker-compose-override.yaml

@@ -0,0 +1,5 @@
+services:
+
+  src-serve-git:
+    # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network
+    container_name: src-serve-git-ubuntu.local

repo-converter/build/docker-compose.yaml

@@ -1,16 +1,22 @@
 version: '2.4'
 
+include:
+  - path:
+    - ../../config/docker-compose-common-services.yaml
+    - docker-compose-override.yaml
+
 services:
 
   repo-converter:
     container_name: repo-converter
-    image: implementation-bridge-repo-converter:build
     build:
       context: .
+    image: sourcegraph/implementation-bridge-repo-converter:build
+    restart: always
     volumes:
       - ../../config/repos-to-convert.yaml:/sourcegraph/repos-to-convert.yaml:ro
       - ../../config/toprc:/root/.config/procps/toprc
       - ../../src-serve-root/:/sourcegraph/src-serve-root
     environment:
       - BRIDGE_REPO_CONVERTER_INTERVAL_SECONDS=10
-      - LOG_LEVEL=DEBUG # DEBUG INFO WARNING ERROR CRITICAL # Default is INFO
+      - LOG_LEVEL=DEBUG # DEBUG INFO WARNING ERROR CRITICAL # Default is INFO

repo-converter/build/run.py

@@ -193,9 +193,11 @@ def parse_repos_to_convert_file_into_repos_dict():
         # Open the file
         with open(args_dict["repos_to_convert_file"], "r") as repos_to_convert_file:
 
-            # Returns a list, not a dict
+            # This should return a dict
             code_hosts_list_temp = yaml.safe_load(repos_to_convert_file)
 
+            # Weird thing we have to do
+            # Reading directory into repos_dict doesn't persist the dict outside the function
             for repo_dict_key in code_hosts_list_temp.keys():
 
                 # Store the repo_dict_key in the repos_dict
@@ -223,7 +225,7 @@ def clone_svn_repos():
         if repos_dict[repo_key].get('type','').lower() != 'svn':
             continue
 
-        # Get config parameters read from repos-to-clone.yaml
+        # Get config parameters read from repos-to-clone.yaml, and set defaults if they're not provided
         svn_repo_code_root      = repos_dict[repo_key].get('svn-repo-code-root','')
         username                = repos_dict[repo_key].get('username','')
         password                = repos_dict[repo_key].get('password','')
@@ -241,6 +243,12 @@ def clone_svn_repos():
 
         ## Parse config parameters into command args
         # TODO: Interpret code_host_name, git_org_name, and git_repo_name if not given
+            # ex. https://svn.apache.org/repos/asf/parquet/site
+            # code_host_name            = svn.apache.org    # can get by removing url scheme, if any, till the first /
+            # arbitrary path on server  = repos             # optional, can either be a directory, or may actually be the repo
+            # git_org_name              = asf
+            # git_repo_name             = parquet
+            # git repo root             = site              # arbitrary path inside the repo where contributors decided to start storing /trunk /branches /tags and other files to be included in the repo
         repo_path = str(args_dict["repo_share_path"]+"/"+code_host_name+"/"+git_org_name+"/"+git_repo_name)
 
         # States

repo-converter/docker-compose-override.yaml

@@ -0,0 +1,5 @@
+services:
+
+  src-serve-git:
+    # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network
+    container_name: src-serve-git-ubuntu.local

repo-converter/docker-compose.yaml

@@ -1,13 +1,18 @@
 version: '2.4'
 
+include:
+  - path:
+    - ../config/docker-compose-common-services.yaml
+    - docker-compose-override.yaml
+
 services:
 
   repo-converter:
     container_name: repo-converter
     image: ghcr.io/sourcegraph/implementation-bridge-repo-converter:insiders
     volumes:
       - ../config/repos-to-convert.yaml:/sourcegraph/repos-to-convert.yaml:ro
-      - ../src-serve-root/:/sourcegraph/src-serve-root"
+      - ../src-serve-root/:/sourcegraph/src-serve-root
     restart: always
     environment:
       - BRIDGE_REPO_CONVERTER_INTERVAL_SECONDS=3600