From 0ebbcc3c8ae6972adfe466aaeaf38fa58539de21 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 2 Jun 2023 09:15:01 -0700 Subject: [PATCH 01/13] allow setting of gitlab scope from env or settings file, add config dumper --- shub/settings.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/shub/settings.py b/shub/settings.py index 018c8308..72d88f91 100644 --- a/shub/settings.py +++ b/shub/settings.py @@ -668,4 +668,20 @@ def __iter__(self): } if ENABLE_GITLAB_AUTH: # noqa - SOCIAL_AUTH_GITLAB_SCOPE = ["api", "read_user"] + if "SOCIAL_AUTH_GITLAB_SCOPE" not in locals(): + SOCIAL_AUTH_GITLAB_SCOPE=[] + SOCIAL_AUTH_GITLAB_SCOPE += get_sregistry_envar_list("SOCIAL_AUTH_GITLAB_SCOPE") + SOCIAL_AUTH_GITLAB_SCOPE = list(set(SOCIAL_AUTH_GITLAB_SCOPE)) + + +if __name__ == "__main__": + # Either dump all settings, or just the ones specified on the command line + import sys + if len(sys.argv) == 1: + for i in locals().copy().keys(): + if i.isupper(): + print(i,'=',eval(i)) + else: + for i in sys.argv[1:]: + if i in locals().keys(): + print(i,'=',eval(i)) From 3a8e20da18da2b1d6aaa8544865296d81bb13dba Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 2 Jun 2023 09:44:11 -0700 Subject: [PATCH 02/13] fix the startup script, since config.py is gone now" --- run_uwsgi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_uwsgi.sh b/run_uwsgi.sh index 04180a00..a0376c30 100755 --- a/run_uwsgi.sh +++ b/run_uwsgi.sh @@ -11,7 +11,7 @@ python manage.py migrate python manage.py collectstatic --noinput service cron start -if grep -Fxq "PLUGINS_ENABLED+=[\"globus\"]" /code/shub/settings/config.py +if python code/shub/settings.py PLUGINS_ENABLED | grep -q globus then # When configured, we can start the endpoint echo "Starting Globus Connect Personal" From bdaec2b0e04754dc2dc704b13161bb8373783e52 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 2 Jun 2023 13:52:15 -0700 Subject: [PATCH 03/13] Update the documentation, and other scripts to reflect settings re-org --- docs/_docs/install/https.md | 10 +++++----- docs/_docs/install/server.md | 22 ++++++++++----------- docs/_docs/plugins/README.md | 6 +++--- docs/_docs/plugins/globus/README.md | 4 ++-- docs/_docs/plugins/google_build/README.md | 20 +++++++++---------- docs/_docs/plugins/ldap/README.md | 12 ++++++------ docs/_docs/plugins/pam/README.md | 18 ++++++++--------- docs/_docs/plugins/pgp/README.md | 2 +- docs/_docs/plugins/saml/README.md | 24 +++++++++++------------ scripts/globus/globus-setup.sh | 8 ++++---- shub/dummy-settings.yaml | 7 ++++++- shub/dummy_secrets.py | 18 ++++++++--------- shub/settings.py | 4 ++-- 13 files changed, 79 insertions(+), 76 deletions(-) diff --git a/docs/_docs/install/https.md b/docs/_docs/install/https.md index d2faca84..1260715f 100644 --- a/docs/_docs/install/https.md +++ b/docs/_docs/install/https.md @@ -83,13 +83,13 @@ The certificate is at "./singularity-registry.org+5.pem" and the key at "./singu It will expire on 29 August 2023 🗓 ``` -Then I moved them into the registry root, and updated my shub/settings/config.py to use +Then I moved them into the registry root, and updated my settings.yaml to use https on localhost. -```python -DOMAIN_NAME = "https://127.0.0.1" -DOMAIN_NAME_HTTP = "https://127.0.0.1" -DOMAIN_NAKED = DOMAIN_NAME_HTTP.replace("https://", "") +```yaml +DOMAIN_NAME: "https://127.0.0.1" +DOMAIN_NAME_HTTP: "https://127.0.0.1" +DOMAIN_NAKED: DOMAIN_NAME_HTTP.replace("https://", "") ``` Finally, we need to make sure that we are using the docker compose file for https, diff --git a/docs/_docs/install/server.md b/docs/_docs/install/server.md index 874cf8bf..080cadd7 100644 --- a/docs/_docs/install/server.md +++ b/docs/_docs/install/server.md @@ -44,7 +44,7 @@ If you don't care about user experience during updates and server downtime, you ## Custom Domain -In the [config settings file](https://github.com/singularityhub/sregistry/blob/master/shub/settings/config.py#L30) +In the [config settings file](https://github.com/singularityhub/sregistry/blob/master/shub/dummy-settings.yaml#L20) you'll find a section for domain names, and other metadata about your registry. You will need to update this to be a custom hostname that you use, and custom names and unique resource identifiers for your registry. For example, if you have a Google Domain and are using Google Cloud, you should be able to set it up using [Cloud DNS](https://console.cloud.google.com/net-services/dns/api/enable?nextPath=%2Fzones&project=singularity-static-registry&authuser=1). Usually this means @@ -130,19 +130,17 @@ more details about credentials in [the Minio documentation](https://docs.min.io/ The `.minio-env` file is also bound to the uwsgi container, so that the generation of the minio storage can be authenticated by the uwsgi container, which is the interface between the Singularity client and minio. For variables that aren't secrets, you can look -in `shub/settings/config.py` and look for the "Storage" section with various +in `settings.yaml` and look for the "Storage" section with various minio variables: -```python -MINIO_SERVER = "minio:9000" # Internal to sregistry -MINIO_EXTERNAL_SERVER = ( - "127.0.0.1:9000" # minio server for Singularity to interact with -) -MINIO_BUCKET = "sregistry" -MINIO_SSL = False # use SSL for minio -MINIO_SIGNED_URL_EXPIRE_MINUTES = 5 -MINIO_REGION = "us-east-1" -MINIO_MULTIPART_UPLOAD = True +```yaml +MINIO_SERVER: "minio:9000" # Internal to sregistry +MINIO_EXTERNAL_SERVER: "127.0.0.1:9000" # minio server for Singularity to interact with +MINIO_BUCKET: "sregistry" +MINIO_SSL: False # use SSL for minio +MINIO_SIGNED_URL_EXPIRE_MINUTES: 5 +MINIO_REGION: "us-east-1" +MINIO_MULTIPART_UPLOAD: True ``` Since the container networking space is different from what the external diff --git a/docs/_docs/plugins/README.md b/docs/_docs/plugins/README.md index da402942..9733c008 100644 --- a/docs/_docs/plugins/README.md +++ b/docs/_docs/plugins/README.md @@ -16,8 +16,8 @@ Plugins distributed with `sregistry` are found in the `shub/plugins` directory. ## Included Plugins The following plugins are included with sregistry, and can be enabled by adding them to the -`PLUGINS_ENABLED` entry in `shub/settings/config.py`. Plugins may require further configuration in -your registries' local `shub/settings/secrets.py` file. +`PLUGINS_ENABLED` entry in `settings.yaml`. Plugins may require further configuration in +your registries' local `shub/secrets.py` file. - [LDAP-Auth](ldap): authentication against LDAP directories - [PAM-Auth](pam): authentication using PAM (unix host users) @@ -54,7 +54,7 @@ Each plugin: - Can register additional context processors by defining a tuple of complete paths to the relevant processors by specifying `CONTEXT_PROCESSORS` in its `__init.py__` - Must provide a documentation file and link in this README. -Plugins are loaded when the plugin name is added to `PLUGINS_ENABLED` in `shub/settings/config.py`. +Plugins are loaded when the plugin name is added to `PLUGINS_ENABLED` in `settings.yaml`. A plugin mentioned here is added to `INSTALLED_APPS` at runtime, and any `AUTHENTICATION_BACKEND` and `CONTEXT_PROCESSORS` listed in the plugin `__init.py__` is merged into the project settings. diff --git a/docs/_docs/plugins/globus/README.md b/docs/_docs/plugins/globus/README.md index 7b7173bd..20307a91 100644 --- a/docs/_docs/plugins/globus/README.md +++ b/docs/_docs/plugins/globus/README.md @@ -12,7 +12,7 @@ The `globus` plugin allows a logged in user to connect their Globus account to a ## Setup -In your `shub/settings/secrets.py` file you need to add a client id and secret generated at [https://developers.globus.org/](https://developers.globus.org/). Navigate to the site and do the following: +In your `shub/secrets.py` file you need to add a client id and secret generated at [https://developers.globus.org/](https://developers.globus.org/). Navigate to the site and do the following: - Click on the first option, "Register your app with Globus" - In the top right click "Add --> New App" @@ -37,7 +37,7 @@ http://localhost/globus/login/ http://localhost/globus/login ``` For reference, we are following [these steps](http://globus-sdk-python.readthedocs.io/en/stable/tutorial/#step-1-get-a-client). -Then click "Create app." Once you have the application created, you should copy the client secret and id, and add to your `shub/settings/secrets.py` file like so: +Then click "Create app." Once you have the application created, you should copy the client secret and id, and add to your `shub/secrets.py` file like so: ``` SOCIAL_AUTH_GLOBUS_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxx" diff --git a/docs/_docs/plugins/google_build/README.md b/docs/_docs/plugins/google_build/README.md index cb9a2fec..ad3b0e3b 100644 --- a/docs/_docs/plugins/google_build/README.md +++ b/docs/_docs/plugins/google_build/README.md @@ -24,16 +24,16 @@ an endpoint. ## Configure sregistry By default, google build is disabled. To configure sregistry to -use Google Cloud build and Storage, in settings/config.py you can enable the plugin by +use Google Cloud build and Storage, in settings.yaml you can enable the plugin by uncommenting it from the list here: -```bash -PLUGINS_ENABLED = [ -# 'ldap_auth', -# 'saml_auth', -# 'globus', - 'google_build' -] +```yaml +PLUGINS_ENABLED: +# - ldap_auth +# - saml_auth +# - pam_auth +# - globus + - google_build ``` You will need to build the image locally with, at least, the build argument ENABLE_GOOGLEBUILD set to true: @@ -43,8 +43,8 @@ $ docker build --build-arg ENABLE_GOOGLEBUILD=true -t ghcr.io/singularityhub/sre ## Secrets -Next, set the following variables in `shub/settings/secrets.py`, -that you can create from `dummy_secrets.py` in the shub/settings folder. +Next, set the following variables in `shub/secrets.py`, +that you can create from `dummy_secrets.py` in the `shub/` folder. The first two speak for themselves, your project name and path to your Google Application Credentials. diff --git a/docs/_docs/plugins/ldap/README.md b/docs/_docs/plugins/ldap/README.md index b8c49dff..e6a84aaa 100644 --- a/docs/_docs/plugins/ldap/README.md +++ b/docs/_docs/plugins/ldap/README.md @@ -14,12 +14,12 @@ LDAP directory. This supports logins against [Microsoft Active Directory](https: To enable LDAP authentication you must: * Build the docker image with the build argument ENABLE_LDAP set to true - * Add `ldap_auth` to the `PLUGINS_ENABLED` list in `shub/settings/config.py` - * Configure the details of your LDAP directory in `shub/settings/secrets.py`. See - `shub/settings/dummy_secrets.py` for an example OpenLDAP configuration. A good start is to do the following: + * Add `ldap_auth` to the `PLUGINS_ENABLED` list in `settings.yaml` + * Configure the details of your LDAP directory in `shub/secrets.py`. See + `shub/dummy_secrets.py` for an example OpenLDAP configuration. A good start is to do the following: ``` -cp shub/settings/dummy_secrets.py shub/settings/secrets.py +cp shub/dummy_secrets.py shub/secrets.py ``` Because no two LDAP directories are the same, configuration can be complex and there are no @@ -269,7 +269,7 @@ The IPAddress thus is `172.17.0.2`. Note that you will need this address in the #### Configure sregistry To configure sregistry to authenticate against our LDAP directory we need to set -the following options in `shub/settings/secrets.py`: +the following options in `shub/secrets.py`: ```python import ldap @@ -302,7 +302,7 @@ AUTH_LDAP_USER_FLAGS_BY_GROUP = { } ``` -Also ensure 'ldap_auth' is listed in `PLUGINS_ENABLED` inside `shub/settings/config.py`. +Also ensure 'ldap_auth' is listed in `PLUGINS_ENABLED` inside `settings.yaml`. Finally, you must build the Docker image with the build argument ENABLE_LDAP set to true: ```bash diff --git a/docs/_docs/plugins/pam/README.md b/docs/_docs/plugins/pam/README.md index b2009fbd..061adab2 100644 --- a/docs/_docs/plugins/pam/README.md +++ b/docs/_docs/plugins/pam/README.md @@ -11,7 +11,7 @@ The `pam_auth` plugin allows users to login to sregistry using the unix accounts the host system. To enable PAM authentication you must: - * Add `pam_auth` to the `PLUGINS_ENABLED` list in `shub/settings/config.py` + * Add `pam_auth` to the `PLUGINS_ENABLED` list in `settings.yaml` * Uncomment binds to /etc/shadow and /etc/passwd in `docker compose.yml` * Build the docker image with the build argument ENABLE_PAM set to true More detailed instructions are below. @@ -27,16 +27,16 @@ and each user will still each need to export their token to push. You can read This is the detailed walkthough to set up the PAM AUthentication plugin. -First, uncomment "pam_auth" at the bottom of `shub/settings/config.py` to +First, uncomment "pam_auth" at the bottom of `settings.yaml` to enable the login option. -```bash -PLUGINS_ENABLED = [ -# 'ldap_auth', - 'pam_auth', -# 'globus', -# 'saml_auth' -] +```yaml +PLUGINS_ENABLED: +# - ldap_auth +# - saml_auth +# - pam_auth +# - globus +# - google_build ``` Since we need to get access to users from the host, diff --git a/docs/_docs/plugins/pgp/README.md b/docs/_docs/plugins/pgp/README.md index 4c35813c..15395316 100644 --- a/docs/_docs/plugins/pgp/README.md +++ b/docs/_docs/plugins/pgp/README.md @@ -13,7 +13,7 @@ protocol, meaning that activating the plugin will expose "lookup" and "add" endp To enable the pgp plugin you must: - * Add `pgp` to the `PLUGINS_ENABLED` list in `shub/settings/config.py` + * Add `pgp` to the `PLUGINS_ENABLED` list in `settings.yaml` * Build the docker image with the build argument ENABLE_PGP set to true: ```bash $ docker build --build-arg ENABLE_PGP=true -t ghcr.io/singularityhub/sregistry . diff --git a/docs/_docs/plugins/saml/README.md b/docs/_docs/plugins/saml/README.md index bbe068a5..c8588327 100644 --- a/docs/_docs/plugins/saml/README.md +++ b/docs/_docs/plugins/saml/README.md @@ -11,9 +11,9 @@ The `saml_auth` plugin allows users to authentication with your [SAML provider]( To enable SAML authentication you must: - * Add `saml_auth` to the `PLUGINS_ENABLED` list in `shub/settings/config.py` - * Add some configuration details to `shub/settings/config.py` - * Configure the details of your SAML provider in in `shub/settings/secrets.py` per instructions provided [here](http://python-social-auth.readthedocs.io/en/latest/backends/saml.html). + * Add `saml_auth` to the `PLUGINS_ENABLED` list in `settings.yaml` + * Add some configuration details to `settings.yaml` + * Configure the details of your SAML provider in in `shub/secrets.py` per instructions provided [here](http://python-social-auth.readthedocs.io/en/latest/backends/saml.html). * Build the docker image with the build argument ENABLE_SAML set to true: ```bash $ docker build --build-arg ENABLE_SAML=true -t ghcr.io/singularityhub/sregistry . @@ -23,7 +23,7 @@ To enable SAML authentication you must: If you haven't yet created a secrets.py, a good start is to do the following: ``` -cp shub/settings/dummy_secrets.py shub/settings/secrets.py +cp shub/dummy_secrets.py shub/secrets.py ``` @@ -31,22 +31,22 @@ cp shub/settings/dummy_secrets.py shub/settings/secrets.py This quick start is intended to demonstrate basic functionality of the SAML authentication. -#### Edit Config.py +#### Edit settings.yaml -In the file `shub/settings/config.py` you should add the name of your institution (used to render the button) +In the file `settings.yaml` you should add the name of your institution (used to render the button) along with the idp (the unique identifier for your SAML server request). That means uncommenting these lines. -```bash -# AUTH_SAML_IDP = "stanford" -# AUTH_SAML_INSTITUTION = "Stanford University" +```yaml +# AUTH_SAML_IDP: "stanford" +# AUTH_SAML_INSTITUTION: "Stanford University" ``` so they appear like: -```bash -AUTH_SAML_IDP = "stanford" -AUTH_SAML_INSTITUTION = "Stanford University" +```yaml +AUTH_SAML_IDP: "stanford" +AUTH_SAML_INSTITUTION: "Stanford University" ``` #### Setting up SAML Auth diff --git a/scripts/globus/globus-setup.sh b/scripts/globus/globus-setup.sh index 09af04b4..586ac526 100644 --- a/scripts/globus/globus-setup.sh +++ b/scripts/globus/globus-setup.sh @@ -21,17 +21,17 @@ token=$(globus endpoint create --personal "${ENDPOINT}" --jmespath 'globus_conne # Export that globus plugin is enabled to config -if ! grep -q \"globus\" /code/shub/settings/config.py; then - echo "PLUGINS_ENABLED+=[\"globus\"]" >> /code/shub/settings/config.py +if ! python code/shub/settings.py PLUGINS_ENABLED | grep -q globus; then + echo "PLUGINS_ENABLED: [\"globus\"]" >> /code/settings.yaml fi # Even if we already have a previous robot name, it must correspond # to naming of this endpoint, so we re-generate (and get a new log file) -echo "ROBOTNAME='${ROBOTNAME}'" >> /code/shub/settings/config.py +echo "ROBOTNAME: '${ROBOTNAME}'" >> /code/settings.yaml ENDPOINT_ID=$(globus endpoint local-id) if [ "${ENDPOINT_ID}" != "No Globus Connect Personal installation found." ]; then - echo "PLUGIN_GLOBUS_ENDPOINT=\"${ENDPOINT_ID}\"" >> /code/shub/settings/config.py + echo "PLUGIN_GLOBUS_ENDPOINT: \"${ENDPOINT_ID}\"" >> /code/settings.yaml fi # Have we set up config paths yet? diff --git a/shub/dummy-settings.yaml b/shub/dummy-settings.yaml index 86fd288e..78ef035c 100644 --- a/shub/dummy-settings.yaml +++ b/shub/dummy-settings.yaml @@ -159,6 +159,11 @@ API_DEFAULT_PAGINATION_CLASS: rest_framework.pagination.LimitOffsetPagination API_PAGE_SIZE: 10 # PLUGINS +# PLUGINS_ENABLED: +# - ldap_auth +# - saml_auth +# - globus +# - google_build # Google Cloud Build + Storage: configure a custom builder and storage endpoint @@ -219,7 +224,7 @@ API_PAGE_SIZE: 10 # OR cn=sregistry_admin,ou=groups,dc=example,dc=com # Globus Assocation (globus) -# Only required if 'globus' is added to PLUGINS_ENABLED in config.py +# Only required if 'globus' is added to PLUGINS_ENABLED in settings.yaml # SOCIAL_AUTH_GLOBUS_KEY: xxxxxxxxxxxx # SOCIAL_AUTH_GLOBUS_USERNAME: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@clients.auth.globus.org # SOCIAL_AUTH_GLOBUS_SECRET: xxxxxxxxxxxxxxxx diff --git a/shub/dummy_secrets.py b/shub/dummy_secrets.py index 7170978e..0abfc21b 100644 --- a/shub/dummy_secrets.py +++ b/shub/dummy_secrets.py @@ -8,18 +8,18 @@ # ============================================================================= # Social Authentication # Set keys and secrets for social authentication methods that you have -# enabled in config.py. +# enabled in settings.yaml. # See https://singularityhub.github.io/sregistry/install.html for full details # ============================================================================= # Twitter OAuth2 -# Only required if ENABLE_TWITTER_AUTH=TRUE in config.py +# Only required if ENABLE_TWITTER_AUTH=TRUE in settings.yaml # SOCIAL_AUTH_TWITTER_KEY = '' # SOCIAL_AUTH_TWITTER_SECRET = '' # ----------------------------------------------------------------------------- # Google OAuth2 -# Only required if ENABLE_GOOGLE_AUTH=TRUE in config.py +# Only required if ENABLE_GOOGLE_AUTH=TRUE in settings.yaml # GOOGLE_CLIENT_FILE='/code/.grilledcheese.json' @@ -35,7 +35,7 @@ # } # ----------------------------------------------------------------------------- # GitHub OAuth -# Only required if ENABLE_GITHUB_AUTH=TRUE in config.py +# Only required if ENABLE_GITHUB_AUTH=TRUE in settings.yaml # http://psa.matiasaguirre.net/docs/backends/github.html?highlight=github # SOCIAL_AUTH_GITHUB_KEY = '' @@ -47,7 +47,7 @@ # ----------------------------------------------------------------------------- # GitHub Enterprise OAuth -# Only required if ENABLE_GITHUB_ENTERPRISE_AUTH=True in config.py +# Only required if ENABLE_GITHUB_ENTERPRISE_AUTH=True in settings.yaml # See https://python-social-auth.readthedocs.io/en/latest/backends/github_enterprise.html # SOCIAL_AUTH_GITHUB_ENTERPRISE_URL = "" @@ -112,11 +112,11 @@ # ============================================================================= # Plugin Authentication -# Set options for authentication plugins that you have enabled in config.py +# Set options for authentication plugins that you have enabled in settings.yaml # ============================================================================= # LDAP Authentication (ldap-auth) -# Only required if 'ldap-auth' is added to PLUGINS_ENABLED in config.py +# Only required if 'ldap-auth' is added to PLUGINS_ENABLED in settings.yaml # This example assumes you are using an OpenLDAP directory # If using an alternative directory - e.g. Microsoft AD, 389 you @@ -167,7 +167,7 @@ # } # Globus Assocation (globus) -# Only required if 'globus' is added to PLUGINS_ENABLED in config.py +# Only required if 'globus' is added to PLUGINS_ENABLED in settings.yaml # SOCIAL_AUTH_GLOBUS_KEY="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # SOCIAL_AUTH_GLOBUS_USERNAME="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@clients.auth.globus.org" @@ -176,7 +176,7 @@ # SAML Authentication (saml) -# Only required if 'saml_auth' is added to PLUGINS_ENABLED in config.py +# Only required if 'saml_auth' is added to PLUGINS_ENABLED in settings.yaml # AUTH_SAML_IDP = "stanford" # AUTH_SAML_INSTITUTION = "Stanford University" diff --git a/shub/settings.py b/shub/settings.py index 72d88f91..d3d03332 100644 --- a/shub/settings.py +++ b/shub/settings.py @@ -144,7 +144,7 @@ "SOCIAL_AUTH_BITBUCKET_OAUTH2_KEY": None, # '' "SOCIAL_AUTH_BITBUCKET_OAUTH2_SECRET": None, # '' # LDAP Authentication (ldap-auth) - # Only required if 'ldap-auth' is added to PLUGINS_ENABLED in config.py + # Only required if 'ldap-auth' is added to PLUGINS_ENABLED in settings.yaml # This example assumes you are using an OpenLDAP directory # If using an alternative directory - e.g. Microsoft AD, 389 you # will need to modify attribute names/mappings accordingly @@ -162,7 +162,7 @@ # Anyone in this group is a superuser for the app "AUTH_LDAP_SUPERUSER_GROUP_FLAGS": None, # "cn=superuser,ou=django,ou=groups,dc=example,dc=com" # Globus Assocation (globus) - # Only required if 'globus' is added to PLUGINS_ENABLED in config.py + # Only required if 'globus' is added to PLUGINS_ENABLED in settings.yaml "SOCIAL_AUTH_GLOBUS_KEY": None, "SOCIAL_AUTH_GLOBUS_USERNAME": None, # "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@clients.auth.globus.org" "SOCIAL_AUTH_GLOBUS_SECRET": None, From 5477171c8a042344fa2b01be97ba2cb534ba0e91 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 2 Jun 2023 14:11:59 -0700 Subject: [PATCH 04/13] move settings debug into a script --- scripts/show_settings.py | 15 +++++++++++++++ shub/settings.py | 13 ------------- 2 files changed, 15 insertions(+), 13 deletions(-) create mode 100755 scripts/show_settings.py diff --git a/scripts/show_settings.py b/scripts/show_settings.py new file mode 100755 index 00000000..77d6c187 --- /dev/null +++ b/scripts/show_settings.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +import sys +import shub.settings as cfg + +if __name__ == "__main__": + # Either dump all settings, or just the ones specified on the command line + if len(sys.argv) == 1: + for key, val in cfg.__dict__.items(): + if key.isupper(): + print(key,'=',val) + else: + for i in sys.argv[1:]: + if i in dir(cfg): + print(i,'=',cfg.__dict__[i]) diff --git a/shub/settings.py b/shub/settings.py index 72d88f91..1748ad66 100644 --- a/shub/settings.py +++ b/shub/settings.py @@ -672,16 +672,3 @@ def __iter__(self): SOCIAL_AUTH_GITLAB_SCOPE=[] SOCIAL_AUTH_GITLAB_SCOPE += get_sregistry_envar_list("SOCIAL_AUTH_GITLAB_SCOPE") SOCIAL_AUTH_GITLAB_SCOPE = list(set(SOCIAL_AUTH_GITLAB_SCOPE)) - - -if __name__ == "__main__": - # Either dump all settings, or just the ones specified on the command line - import sys - if len(sys.argv) == 1: - for i in locals().copy().keys(): - if i.isupper(): - print(i,'=',eval(i)) - else: - for i in sys.argv[1:]: - if i in locals().keys(): - print(i,'=',eval(i)) From 5d08344a83aeef3665e713b07acdeaf5539f8333 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 2 Jun 2023 15:49:21 -0700 Subject: [PATCH 05/13] move settings script to a django management script --- scripts/show_settings.py | 15 --------- .../base/management/commands/show_settings.py | 31 +++++++++++++++++++ 2 files changed, 31 insertions(+), 15 deletions(-) delete mode 100755 scripts/show_settings.py create mode 100755 shub/apps/base/management/commands/show_settings.py diff --git a/scripts/show_settings.py b/scripts/show_settings.py deleted file mode 100755 index 77d6c187..00000000 --- a/scripts/show_settings.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import shub.settings as cfg - -if __name__ == "__main__": - # Either dump all settings, or just the ones specified on the command line - if len(sys.argv) == 1: - for key, val in cfg.__dict__.items(): - if key.isupper(): - print(key,'=',val) - else: - for i in sys.argv[1:]: - if i in dir(cfg): - print(i,'=',cfg.__dict__[i]) diff --git a/shub/apps/base/management/commands/show_settings.py b/shub/apps/base/management/commands/show_settings.py new file mode 100755 index 00000000..6c562dd1 --- /dev/null +++ b/shub/apps/base/management/commands/show_settings.py @@ -0,0 +1,31 @@ +""" + +Copyright 2017-2023 Evan Felix. + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed +with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" + +import shub.settings as cfg +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + requires_system_checks = [] + help = """Show configured setting""" + + def add_arguments(self, parser): + parser.add_argument("setting", nargs="*") + + def handle(self, *args, **kwargs): + # Either dump all settings, or just the ones specified on the command line + if len(kwargs['setting']) == 0: + for key, val in cfg.__dict__.items(): + if key.isupper(): + print(key,'=',val) + else: + for i in kwargs['setting']: + if i in dir(cfg): + print(i,'=',cfg.__dict__[i]) From f55262229ca07203229d77266f0a43bdd6c24d42 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 2 Jun 2023 16:10:11 -0700 Subject: [PATCH 06/13] rework list defaults a little so that is matches other defaults --- shub/settings.py | 82 ++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/shub/settings.py b/shub/settings.py index 1748ad66..e703e500 100644 --- a/shub/settings.py +++ b/shub/settings.py @@ -202,6 +202,32 @@ "CONTAINER_SIGNED_URL_EXPIRE_SECONDS": None, # 10 } +LIST_DEFAULTS = { + #list the scopes that will be needed by the gitlab OAuth provider + "SOCIAL_AUTH_GITLAB_SCOPE": [], + + # Plugins + # Add the name of a plugin under shub.plugins here to enable it + + # Available Plugins: + + # - ldap_auth: Allows sregistry to authenticate against an LDAP directory + # - google_build: a custom storage with that uses Google Cloud Build + Storage + # - pam_auth: Allow users from (docker) host to log in + # - globus: allows connection from sregistry to endpoints + # - saml_auth: authentication with SAML + # - pgp: deploy a key server alongside your registry + + "PLUGINS_ENABLED": [ + # 'pgp' + # 'ldap_auth', + # 'google_build' + # 'pam_auth', + # 'globus', + # 'saml_auth' + ] +} + # Environment helpers @@ -252,6 +278,11 @@ def get_sregistry_envar_list(key): STRING_DEFAULTS[key] = value +for key in LIST_DEFAULTS: + value = get_sregistry_envar_list(key) + if value is not []: + LIST_DEFAULTS[key] = list(set(value)) + # Finally, create settings object class Settings: def __init__(self, dictionary): @@ -269,7 +300,7 @@ def __iter__(self): yield key, value -DEFAULTS = STRING_DEFAULTS | BOOLEAN_DEFAULTS | INTEGER_DEFAULTS +DEFAULTS = STRING_DEFAULTS | BOOLEAN_DEFAULTS | INTEGER_DEFAULTS | LIST_DEFAULTS # If we have a settings file, it takes preference to DEFAULTS if os.path.exists(SETTINGS_FILE): @@ -426,33 +457,6 @@ def __iter__(self): MINIO_ROOT_USER = os.environ.get("MINIO_ROOT_USER") or cfg.MINIO_ROOT_USER MINIO_ROOT_PASSWORD = os.environ.get("MINIO_ROOT_PASSWORD") or cfg.MINIO_ROOT_PASSWORD -# Plugins -# Add the name of a plugin under shub.plugins here to enable it - -# Available Plugins: - -# - ldap_auth: Allows sregistry to authenticate against an LDAP directory -# - google_build: a custom storage with that uses Google Cloud Build + Storage -# - pam_auth: Allow users from (docker) host to log in -# - globus: allows connection from sregistry to endpoints -# - saml_auth: authentication with SAML -# - pgp: deploy a key server alongside your registry - -PLUGINS_ENABLED = [ - # 'pgp' - # 'ldap_auth', - # 'google_build' - # 'pam_auth', - # 'globus', - # 'saml_auth' -] - -# Any plugins enabled from the environment? -PLUGINS_ENABLED += get_sregistry_envar_list("PLUGINS_ENABLED") - -# Ensure unique set -PLUGINS_ENABLED = list(set(PLUGINS_ENABLED)) - # Default Django logging is WARNINGS+ to console # so visible via docker-compose logs uwsgi LOGGING = { @@ -559,6 +563,15 @@ def __iter__(self): RQ = {"host": cfg.REDIS_HOST, "db": 0} + +# Finally, ensure all variables in cfg are set in locals +for key, value in cfg: + # Don't set if the value is empty, or it's been set previously + if value is None or key in locals() and locals()[key] is not None: + continue + locals()[key] = value + + # Plugins # If PAM_AUTH in plugins enbled, add django_pam @@ -633,13 +646,6 @@ def __iter__(self): for context_processor in plugin.CONTEXT_PROCESSORS: TEMPLATES[0]["OPTIONS"]["context_processors"].append(context_processor) -# Finally, ensure all variables in cfg are set in locals -for key, value in cfg: - # Don't set if the value is empty, or it's been set previously - if value is None or key in locals() and locals()[key] is not None: - continue - locals()[key] = value - # Try reading in from secrets first (no issue if not found) try: from .secrets import * # noqa @@ -666,9 +672,3 @@ def __iter__(self): "access_type": "offline", "approval_prompt": "auto", } - -if ENABLE_GITLAB_AUTH: # noqa - if "SOCIAL_AUTH_GITLAB_SCOPE" not in locals(): - SOCIAL_AUTH_GITLAB_SCOPE=[] - SOCIAL_AUTH_GITLAB_SCOPE += get_sregistry_envar_list("SOCIAL_AUTH_GITLAB_SCOPE") - SOCIAL_AUTH_GITLAB_SCOPE = list(set(SOCIAL_AUTH_GITLAB_SCOPE)) From 6d8ad6f2e3a000abc51b4351185e65e0afee4c02 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Mon, 5 Jun 2023 17:32:27 -0700 Subject: [PATCH 07/13] remove header as the MPL does not require them in all files --- shub/apps/base/management/commands/show_settings.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/shub/apps/base/management/commands/show_settings.py b/shub/apps/base/management/commands/show_settings.py index 6c562dd1..034008e3 100755 --- a/shub/apps/base/management/commands/show_settings.py +++ b/shub/apps/base/management/commands/show_settings.py @@ -1,13 +1,3 @@ -""" - -Copyright 2017-2023 Evan Felix. - -This Source Code Form is subject to the terms of the -Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed -with this file, You can obtain one at http://mozilla.org/MPL/2.0/. - -""" - import shub.settings as cfg from django.core.management.base import BaseCommand From b9718e239397c0c7265b1de601f47590c7b3f702 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Tue, 6 Jun 2023 08:45:53 -0700 Subject: [PATCH 08/13] Just overwrite the values, if there is one --- shub/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shub/settings.py b/shub/settings.py index e703e500..0a97a661 100644 --- a/shub/settings.py +++ b/shub/settings.py @@ -280,7 +280,7 @@ def get_sregistry_envar_list(key): for key in LIST_DEFAULTS: value = get_sregistry_envar_list(key) - if value is not []: + if value: LIST_DEFAULTS[key] = list(set(value)) # Finally, create settings object From 8a4c8e32be749005c3378c4210f9d859dc4c1473 Mon Sep 17 00:00:00 2001 From: Evan Felix Date: Tue, 6 Jun 2023 12:57:35 -0700 Subject: [PATCH 09/13] change version for new settings --- CHANGELOG.md | 1 + VERSION | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c04b6c16..1e9ecf42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ represented by the pull requests that fixed them. Critical items to know are: ## [master](https://github.com/singularityhub/sregistry/tree/master) (master) + - setup LIST_DEFAULTS for list settings. (2.1.1) - update to use Django 4.2 (2.1.0) - consolidate config into one file with environment (2.0.0) - This is an API breaking change, as the settings are completely refactored diff --git a/VERSION b/VERSION index 7ec1d6db..3e3c2f1e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 +2.1.1 From d6c4cec2c446a454eca7b8a5834946211f55c11b Mon Sep 17 00:00:00 2001 From: Evan Felix Date: Tue, 6 Jun 2023 13:01:34 -0700 Subject: [PATCH 10/13] adjust scripts to reflect changes in #435 --- CHANGELOG.md | 1 + run_uwsgi.sh | 2 +- scripts/globus/globus-setup.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) mode change 100644 => 100755 scripts/globus/globus-setup.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e9ecf42..c4e21e7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ represented by the pull requests that fixed them. Critical items to know are: ## [master](https://github.com/singularityhub/sregistry/tree/master) (master) - setup LIST_DEFAULTS for list settings. (2.1.1) + - update docs referring to settings, and fix scripts using settings - update to use Django 4.2 (2.1.0) - consolidate config into one file with environment (2.0.0) - This is an API breaking change, as the settings are completely refactored diff --git a/run_uwsgi.sh b/run_uwsgi.sh index a0376c30..dace8c3d 100755 --- a/run_uwsgi.sh +++ b/run_uwsgi.sh @@ -11,7 +11,7 @@ python manage.py migrate python manage.py collectstatic --noinput service cron start -if python code/shub/settings.py PLUGINS_ENABLED | grep -q globus +if python manage.py show_settings PLUGINS_ENABLED | grep -q globus then # When configured, we can start the endpoint echo "Starting Globus Connect Personal" diff --git a/scripts/globus/globus-setup.sh b/scripts/globus/globus-setup.sh old mode 100644 new mode 100755 index 586ac526..51553394 --- a/scripts/globus/globus-setup.sh +++ b/scripts/globus/globus-setup.sh @@ -21,7 +21,7 @@ token=$(globus endpoint create --personal "${ENDPOINT}" --jmespath 'globus_conne # Export that globus plugin is enabled to config -if ! python code/shub/settings.py PLUGINS_ENABLED | grep -q globus; then +if ! python manage.py show_settings PLUGINS_ENABLED | grep -q globus; then echo "PLUGINS_ENABLED: [\"globus\"]" >> /code/settings.yaml fi From 53c83476c61eb4246a402b3ea145b2a8fc57da0b Mon Sep 17 00:00:00 2001 From: Evan Felix Date: Tue, 6 Jun 2023 13:52:25 -0700 Subject: [PATCH 11/13] cleanup some flake8 issues --- shub/settings.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/shub/settings.py b/shub/settings.py index ed0651b8..8288b522 100644 --- a/shub/settings.py +++ b/shub/settings.py @@ -283,6 +283,7 @@ def get_sregistry_envar_list(key): if value: LIST_DEFAULTS[key] = list(set(value)) + # Finally, create settings object class Settings: def __init__(self, dictionary): @@ -563,7 +564,6 @@ def __iter__(self): RQ = {"host": cfg.REDIS_HOST, "db": 0} - # Finally, ensure all variables in cfg are set in locals for key, value in cfg: # Don't set if the value is empty, or it's been set previously @@ -575,11 +575,11 @@ def __iter__(self): # Plugins # If PAM_AUTH in plugins enbled, add django_pam -if "pam_auth" in PLUGINS_ENABLED: +if "pam_auth" in PLUGINS_ENABLED: # noqa INSTALLED_APPS += ["django_pam"] # If LDAP_AUTH in plugins enabled, populate from settings -if "ldap_auth" in PLUGINS_ENABLED: +if "ldap_auth" in PLUGINS_ENABLED: # noqa # To work with OpenLDAP and posixGroup groups we need to import some things import ldap from django_auth_ldap.config import LDAPSearch, PosixGroupType @@ -615,7 +615,7 @@ def __iter__(self): # If google_build in use, we are required to include GitHub -if "google_build" in PLUGINS_ENABLED: +if "google_build" in PLUGINS_ENABLED: # noqa # For task discovery by celery SOCIAL_AUTH_GITHUB_SCOPE = [ "admin:repo_hook", @@ -628,7 +628,7 @@ def __iter__(self): ENABLE_GITHUB_AUTH = True # Apply any plugin settings -for plugin in PLUGINS_ENABLED: +for plugin in PLUGINS_ENABLED: # noqa plugin_module = "shub.plugins." + plugin plugin = import_module(plugin_module) From daba6910b3c9548b4584b55fa1a75862f763ea32 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 9 Jun 2023 08:54:00 -0700 Subject: [PATCH 12/13] updates for pre-commit --- shub/settings.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/shub/settings.py b/shub/settings.py index 8288b522..bc6b21db 100644 --- a/shub/settings.py +++ b/shub/settings.py @@ -203,21 +203,17 @@ } LIST_DEFAULTS = { - #list the scopes that will be needed by the gitlab OAuth provider + # list the scopes that will be needed by the gitlab OAuth provider "SOCIAL_AUTH_GITLAB_SCOPE": [], - # Plugins # Add the name of a plugin under shub.plugins here to enable it - # Available Plugins: - # - ldap_auth: Allows sregistry to authenticate against an LDAP directory # - google_build: a custom storage with that uses Google Cloud Build + Storage # - pam_auth: Allow users from (docker) host to log in # - globus: allows connection from sregistry to endpoints # - saml_auth: authentication with SAML # - pgp: deploy a key server alongside your registry - "PLUGINS_ENABLED": [ # 'pgp' # 'ldap_auth', @@ -225,7 +221,7 @@ # 'pam_auth', # 'globus', # 'saml_auth' - ] + ], } # Environment helpers @@ -575,11 +571,11 @@ def __iter__(self): # Plugins # If PAM_AUTH in plugins enbled, add django_pam -if "pam_auth" in PLUGINS_ENABLED: # noqa +if "pam_auth" in PLUGINS_ENABLED: # noqa INSTALLED_APPS += ["django_pam"] # If LDAP_AUTH in plugins enabled, populate from settings -if "ldap_auth" in PLUGINS_ENABLED: # noqa +if "ldap_auth" in PLUGINS_ENABLED: # noqa # To work with OpenLDAP and posixGroup groups we need to import some things import ldap from django_auth_ldap.config import LDAPSearch, PosixGroupType @@ -615,7 +611,7 @@ def __iter__(self): # If google_build in use, we are required to include GitHub -if "google_build" in PLUGINS_ENABLED: # noqa +if "google_build" in PLUGINS_ENABLED: # noqa # For task discovery by celery SOCIAL_AUTH_GITHUB_SCOPE = [ "admin:repo_hook", @@ -628,7 +624,7 @@ def __iter__(self): ENABLE_GITHUB_AUTH = True # Apply any plugin settings -for plugin in PLUGINS_ENABLED: # noqa +for plugin in PLUGINS_ENABLED: # noqa plugin_module = "shub.plugins." + plugin plugin = import_module(plugin_module) From f1b18c1d9e2ae81d7f0437aafd28b22db182d8b1 Mon Sep 17 00:00:00 2001 From: "Evan J. Felix" Date: Fri, 9 Jun 2023 11:35:04 -0700 Subject: [PATCH 13/13] more linting issues --- shub/apps/base/management/commands/show_settings.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/shub/apps/base/management/commands/show_settings.py b/shub/apps/base/management/commands/show_settings.py index 034008e3..1e2f01cf 100755 --- a/shub/apps/base/management/commands/show_settings.py +++ b/shub/apps/base/management/commands/show_settings.py @@ -1,6 +1,7 @@ -import shub.settings as cfg from django.core.management.base import BaseCommand +import shub.settings as cfg + class Command(BaseCommand): requires_system_checks = [] @@ -11,11 +12,11 @@ def add_arguments(self, parser): def handle(self, *args, **kwargs): # Either dump all settings, or just the ones specified on the command line - if len(kwargs['setting']) == 0: + if len(kwargs["setting"]) == 0: for key, val in cfg.__dict__.items(): if key.isupper(): - print(key,'=',val) + print(key, "=", val) else: - for i in kwargs['setting']: + for i in kwargs["setting"]: if i in dir(cfg): - print(i,'=',cfg.__dict__[i]) + print(i, "=", cfg.__dict__[i])