From a9946c0e1f4d38880ec7f0dd4d226a623e653fae Mon Sep 17 00:00:00 2001 From: Marc LeBlanc Date: Mon, 19 Feb 2024 15:31:54 +0300 Subject: [PATCH] Refactoring docker compose files for easier execution --- config/docker-compose-common-services.yaml | 27 ++++++++++++++++ host-ubuntu/docker-compose-override.yaml | 5 +++ host-ubuntu/docker-compose.yaml | 31 +++---------------- host-wsl2-ubuntu/docker-compose-override.yaml | 5 +++ host-wsl2-ubuntu/docker-compose.yaml | 31 +++---------------- host-wsl2-ubuntu/pull-build-start.sh | 14 +++++++++ .../build/docker-compose-override.yaml | 5 +++ repo-converter/build/docker-compose.yaml | 10 ++++-- repo-converter/build/run.py | 12 +++++-- repo-converter/docker-compose-override.yaml | 5 +++ repo-converter/docker-compose.yaml | 7 ++++- 11 files changed, 93 insertions(+), 59 deletions(-) create mode 100644 config/docker-compose-common-services.yaml create mode 100644 host-ubuntu/docker-compose-override.yaml create mode 100644 host-wsl2-ubuntu/docker-compose-override.yaml create mode 100644 host-wsl2-ubuntu/pull-build-start.sh create mode 100644 repo-converter/build/docker-compose-override.yaml create mode 100644 repo-converter/docker-compose-override.yaml diff --git a/config/docker-compose-common-services.yaml b/config/docker-compose-common-services.yaml new file mode 100644 index 0000000..b962235 --- /dev/null +++ b/config/docker-compose-common-services.yaml @@ -0,0 +1,27 @@ +version: '2.4' + +services: + + cloud-agent: + container_name: cloud-agent + image: index.docker.io/sourcegraph/src-tunnel-agent:2024-02-15-04-02-110c2ea9 + volumes: + - ../config/cloud-agent-service-account-key.json:/sourcegraph/cloud-agent-service-account-key.json:ro + - ../config/cloud-agent-config.yaml:/sourcegraph/cloud-agent-config.yaml:ro + command: ["-config=/sourcegraph/cloud-agent-config.yaml"] + restart: always + networks: + - sourcegraph + + src-serve-git: + # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network + container_name: src-serve-git-ubuntu.local + image: index.docker.io/sourcegraph/src-cli:latest + volumes: + - ../src-serve-root/:/sourcegraph/src-serve-root:ro + command: "serve-git -addr :443 /sourcegraph/src-serve-root" + networks: + - sourcegraph + +networks: + sourcegraph: diff --git a/host-ubuntu/docker-compose-override.yaml b/host-ubuntu/docker-compose-override.yaml new file mode 100644 index 0000000..16f493e --- /dev/null +++ b/host-ubuntu/docker-compose-override.yaml @@ -0,0 +1,5 @@ +services: + + src-serve-git: + # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network + container_name: src-serve-git-ubuntu.local diff --git a/host-ubuntu/docker-compose.yaml b/host-ubuntu/docker-compose.yaml index c6b0bbb..922be6c 100644 --- a/host-ubuntu/docker-compose.yaml +++ b/host-ubuntu/docker-compose.yaml @@ -1,27 +1,4 @@ -version: '2.4' - -services: - - cloud-agent: - container_name: cloud-agent - image: index.docker.io/sourcegraph/src-tunnel-agent:2024-02-05-22-28-333320ee - volumes: - - ../config/cloud-agent-service-account-key.json:/sourcegraph/cloud-agent-service-account-key.json:ro - - ../config/cloud-agent-config.yaml:/sourcegraph/cloud-agent-config.yaml:ro - command: ["-config=/sourcegraph/cloud-agent-config.yaml"] - restart: always - networks: - - sourcegraph - - src-serve-git: - # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network - container_name: src-serve-git-ubuntu.local - image: index.docker.io/sourcegraph/src-cli:latest - volumes: - - ../src-serve-root/:/sourcegraph/src-serve-root:ro - command: "serve-git -addr :443 /sourcegraph/src-serve-root" - networks: - - sourcegraph - -networks: - sourcegraph: +include: + - path: + - ../config/docker-compose-common-services.yaml + - docker-compose-override.yaml diff --git a/host-wsl2-ubuntu/docker-compose-override.yaml b/host-wsl2-ubuntu/docker-compose-override.yaml new file mode 100644 index 0000000..91210aa --- /dev/null +++ b/host-wsl2-ubuntu/docker-compose-override.yaml @@ -0,0 +1,5 @@ +services: + + src-serve-git: + # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network + container_name: src-serve-git-wsl.local diff --git a/host-wsl2-ubuntu/docker-compose.yaml b/host-wsl2-ubuntu/docker-compose.yaml index 14e9198..922be6c 100644 --- a/host-wsl2-ubuntu/docker-compose.yaml +++ b/host-wsl2-ubuntu/docker-compose.yaml @@ -1,27 +1,4 @@ -version: '2.4' - -services: - - cloud-agent: - container_name: cloud-agent - image: index.docker.io/sourcegraph/src-tunnel-agent:2024-02-15-04-02-110c2ea9 - volumes: - - ../config/cloud-agent-service-account-key.json:/sourcegraph/cloud-agent-service-account-key.json:ro - - ../config/cloud-agent-config.yaml:/sourcegraph/cloud-agent-config.yaml:ro - command: ["-config=/sourcegraph/cloud-agent-config.yaml"] - restart: always - networks: - - sourcegraph - - src-serve-git: - # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network - container_name: src-serve-git-wsl.local - image: index.docker.io/sourcegraph/src-cli:latest - volumes: - - ../src-serve-root/:/sourcegraph/src-serve-root:ro - command: "serve-git -addr :443 /sourcegraph/src-serve-root" - networks: - - sourcegraph - -networks: - sourcegraph: +include: + - path: + - ../config/docker-compose-common-services.yaml + - docker-compose-override.yaml diff --git a/host-wsl2-ubuntu/pull-build-start.sh b/host-wsl2-ubuntu/pull-build-start.sh new file mode 100644 index 0000000..de91976 --- /dev/null +++ b/host-wsl2-ubuntu/pull-build-start.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# To be used in a cronjob to always pull and build the latest +# every 10 minutes +# so that the running container is only 10 minutes behind the latest commit in the branch + +# crontab -e +# */10 * * * * bash /sourcegraph/implementation-bridges/host-wsl2-ubuntu/pull-build-start.sh >> /sourcegraph/implementation-bridges/host-wsl2-ubuntu/pull-build-start.log 2>&1 + +repo_build_path="/sourcegraph/implementation-bridges/repo-converter/build" + +git -C $repo_build_path pull + +docker compose -f $repo_build_path/docker-compose.yaml up -d --build diff --git a/repo-converter/build/docker-compose-override.yaml b/repo-converter/build/docker-compose-override.yaml new file mode 100644 index 0000000..16f493e --- /dev/null +++ b/repo-converter/build/docker-compose-override.yaml @@ -0,0 +1,5 @@ +services: + + src-serve-git: + # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network + container_name: src-serve-git-ubuntu.local diff --git a/repo-converter/build/docker-compose.yaml b/repo-converter/build/docker-compose.yaml index 6573998..355a3d0 100644 --- a/repo-converter/build/docker-compose.yaml +++ b/repo-converter/build/docker-compose.yaml @@ -1,16 +1,22 @@ version: '2.4' +include: + - path: + - ../../config/docker-compose-common-services.yaml + - docker-compose-override.yaml + services: repo-converter: container_name: repo-converter - image: implementation-bridge-repo-converter:build build: context: . + image: sourcegraph/implementation-bridge-repo-converter:build + restart: always volumes: - ../../config/repos-to-convert.yaml:/sourcegraph/repos-to-convert.yaml:ro - ../../config/toprc:/root/.config/procps/toprc - ../../src-serve-root/:/sourcegraph/src-serve-root environment: - BRIDGE_REPO_CONVERTER_INTERVAL_SECONDS=10 - - LOG_LEVEL=DEBUG # DEBUG INFO WARNING ERROR CRITICAL # Default is INFO + - LOG_LEVEL=DEBUG # DEBUG INFO WARNING ERROR CRITICAL # Default is INFO \ No newline at end of file diff --git a/repo-converter/build/run.py b/repo-converter/build/run.py index ae1abb2..70c39f1 100644 --- a/repo-converter/build/run.py +++ b/repo-converter/build/run.py @@ -193,9 +193,11 @@ def parse_repos_to_convert_file_into_repos_dict(): # Open the file with open(args_dict["repos_to_convert_file"], "r") as repos_to_convert_file: - # Returns a list, not a dict + # This should return a dict code_hosts_list_temp = yaml.safe_load(repos_to_convert_file) + # Weird thing we have to do + # Reading directory into repos_dict doesn't persist the dict outside the function for repo_dict_key in code_hosts_list_temp.keys(): # Store the repo_dict_key in the repos_dict @@ -223,7 +225,7 @@ def clone_svn_repos(): if repos_dict[repo_key].get('type','').lower() != 'svn': continue - # Get config parameters read from repos-to-clone.yaml + # Get config parameters read from repos-to-clone.yaml, and set defaults if they're not provided svn_repo_code_root = repos_dict[repo_key].get('svn-repo-code-root','') username = repos_dict[repo_key].get('username','') password = repos_dict[repo_key].get('password','') @@ -241,6 +243,12 @@ def clone_svn_repos(): ## Parse config parameters into command args # TODO: Interpret code_host_name, git_org_name, and git_repo_name if not given + # ex. https://svn.apache.org/repos/asf/parquet/site + # code_host_name = svn.apache.org # can get by removing url scheme, if any, till the first / + # arbitrary path on server = repos # optional, can either be a directory, or may actually be the repo + # git_org_name = asf + # git_repo_name = parquet + # git repo root = site # arbitrary path inside the repo where contributors decided to start storing /trunk /branches /tags and other files to be included in the repo repo_path = str(args_dict["repo_share_path"]+"/"+code_host_name+"/"+git_org_name+"/"+git_repo_name) # States diff --git a/repo-converter/docker-compose-override.yaml b/repo-converter/docker-compose-override.yaml new file mode 100644 index 0000000..16f493e --- /dev/null +++ b/repo-converter/docker-compose-override.yaml @@ -0,0 +1,5 @@ +services: + + src-serve-git: + # Uses a valid hostname as container_name, to trick the cloud agent and code host config into finding this container on the Docker network + container_name: src-serve-git-ubuntu.local diff --git a/repo-converter/docker-compose.yaml b/repo-converter/docker-compose.yaml index 6c045b6..9e4d8b2 100644 --- a/repo-converter/docker-compose.yaml +++ b/repo-converter/docker-compose.yaml @@ -1,5 +1,10 @@ version: '2.4' +include: + - path: + - ../config/docker-compose-common-services.yaml + - docker-compose-override.yaml + services: repo-converter: @@ -7,7 +12,7 @@ services: image: ghcr.io/sourcegraph/implementation-bridge-repo-converter:insiders volumes: - ../config/repos-to-convert.yaml:/sourcegraph/repos-to-convert.yaml:ro - - ../src-serve-root/:/sourcegraph/src-serve-root" + - ../src-serve-root/:/sourcegraph/src-serve-root restart: always environment: - BRIDGE_REPO_CONVERTER_INTERVAL_SECONDS=3600