OZ-573: Superset SSO support + configs assuming Traefik proxy (#37)

.gitignore

@@ -79,6 +79,7 @@ settings.xml
 .idea/**/usage.statistics.xml
 .idea/**/dictionaries
 .idea/**/shelf
+.idea
 
 # AWS User-specific
 .idea/**/aws.xml
@@ -254,4 +255,4 @@ docker/data/parquet/
 !docker/data/parquet/.gitkeep
 data/
 debezium-connect/
-scripts/distro
+scripts/distro

docker/.env

@@ -43,11 +43,9 @@ DATABASE_PORT=5432
 ENABLE_PROXY_FIX=True
 REDIS_HOST=redis
 REDIS_PORT=6379
-SUPERSET_CLIENT_SECRET=
-SUPERSET_CLIENT_ID=superset
 SUPERSET_HOSTNAME=
 ANALYTICS_DATASOURCE_NAME=PostgreSQL
-ENABLE_OAUTH=false
+
 
 ANALYTICS_DB_USER=analytics
 ANALYTICS_DB_PASSWORD=password
@@ -71,7 +69,6 @@ POSTGRES_DB_HOST=postgresql
 SUPERSET_DB=superset
 SUPERSET_DB_USER=superset
 SUPERSET_DB_PASSWORD=password
-ENABLE_OAUTH=false
 
 
 # Flink
@@ -118,6 +115,7 @@ SUPERSET_HOME=
 
 ZOOKEEPER_URL=zookeeper:2181
 
-#Keycloak
-KEYCLOAK_HOSTNAME=
-ISSUER_URL=
+# SSO Support
+ENABLE_OAUTH=false
+SUPERSET_CLIENT_UUID=891b980a-9edb-4c72-a63d-1f8e488d6ad4
+SUPERSET_CLIENT_SECRET=znZK8dvk7hLOpwfU

docker/docker-compose-db.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   mysql:
     networks:
@@ -49,4 +48,4 @@ volumes:
    mysql-data:
    postgresql-data: ~
 networks:
-  ozone-analytics:
+  ozone-analytics:

docker/docker-compose-superset.yaml

@@ -41,9 +41,9 @@ services:
       - ${SUPERSET_CONFIG_PATH}/superset_config.py:/app/superset_config.py
       - ${SUPERSET_CONFIG_PATH}/security.py:/app/security.py
       - ${SUPERSET_CONFIG_PATH}/superset-init.sh:/app/superset-init.sh
-      
+
   superset-worker:
-    command: "celery --app=superset.tasks.celery_app:app worker --pool=gevent -Ofair -n worker1@%h --loglevel=INFO"
+    command: "celery --app=superset.tasks.celery_app:app worker -Ofair -n worker1@%h --loglevel=INFO"
     depends_on:
       redis:
         condition: service_started
@@ -93,4 +93,4 @@ networks:
   ozone-analytics:
   web:
     external: true
-    name: web
+    name: web

docker/superset/config/security.py

@@ -0,0 +1,67 @@
+from superset.security import SupersetSecurityManager
+import logging
+from flask_appbuilder.security.views import AuthOAuthView
+from flask_appbuilder.baseviews import expose
+import os
+from six.moves.urllib.parse import urlencode
+import redis
+import time
+from flask import (
+    redirect,
+    request,
+    g
+)
+
+TOKEN_PREFIX = "oauth_id_token_"
+REDIS_HOST = os.getenv("REDIS_HOST", "redis")
+REDIS_PORT = os.getenv("REDIS_PORT", 6379)
+redis_db = redis.Redis(host=REDIS_HOST, port=REDIS_PORT, decode_responses=True)
+
+class CustomAuthOAuthView(AuthOAuthView):
+
+    @expose("/logout/")
+    def logout(self, provider="keycloak", register=None):
+        user_id = str(g.user.id)
+        provider_obj = self.appbuilder.sm.oauth_remotes[provider]
+        redirect_url = request.url_root.strip('/') + self.appbuilder.get_url_for_login
+        logout_base_url = provider_obj.api_base_url + "logout"
+        params = {
+            "client_id": provider_obj.client_id,
+            "post_logout_redirect_uri": redirect_url
+        }
+        if redis_db.get(TOKEN_PREFIX + user_id):
+            params["id_token_hint"] = redis_db.get(TOKEN_PREFIX + user_id)
+
+        ret = super().logout()
+        time.sleep(1)
+
+        return redirect("{}?{}".format(logout_base_url, urlencode(params)))
+
+
+class CustomSecurityManager(SupersetSecurityManager):
+    # override the logout function
+    authoauthview = CustomAuthOAuthView
+
+    def oauth_user_info(self, provider, response=None):
+        logging.debug("Oauth2 provider: {0}.".format(provider))
+        if provider == 'keycloak':
+            me = self.appbuilder.sm.oauth_remotes[provider].get('userinfo').json()
+            return {
+                "username": me.get("preferred_username", ""),
+                "first_name": me.get("given_name", ""),
+                "last_name": me.get("family_name", ""),
+                "email": me.get("email", ""),
+                'roles': me.get('roles', ['Public']),
+                'id_token': response["id_token"]
+            }
+        return {}
+
+    def auth_user_oauth(self, userinfo):
+        user = super(CustomSecurityManager, self).auth_user_oauth(userinfo)
+        redis_db.set(TOKEN_PREFIX + str(user.id), userinfo["id_token"])
+        del userinfo["id_token"]
+        roles = [self.find_role(x) for x in userinfo['roles']]
+        roles = [x for x in roles if x is not None]
+        user.roles = roles
+        self.update_user(user)
+        return user

docker/superset/config/superset-init.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+STEP_CNT=6
+
+echo_step() {
+cat <<EOF
+######################################################################
+Init Step ${1}/${STEP_CNT} [${2}] -- ${3}
+######################################################################
+EOF
+}
+# Initialize the database
+echo_step "1" "Starting" "Applying DB migrations"
+superset db upgrade
+echo_step "1" "Complete" "Applying DB migrations"
+
+# Create an admin user
+echo_step "2" "Starting" "Setting up admin user ( admin / $ADMIN_PASSWORD )"
+superset fab create-admin \
+              --username admin \
+              --firstname Superset \
+              --lastname Admin \
+              --email [email protected] \
+              --password $ADMIN_PASSWORD
+echo_step "2" "Complete" "Setting up admin user"
+# Create default roles and permissions
+echo_step "3" "Starting" "Setting up roles and perms"
+superset init
+
+echo_step "3" "Complete" "Setting up roles and perms"
+if [ "$SUPERSET_LOAD_EXAMPLES" = "yes" ]; then
+    # Load some data to play with" row_number() over(partition by visit.patient_id order by visit.visit_id) as number_occurences," +
+    echo_step "4" "Starting" "Loading examples"
+    # If Cypress run which consumes superset_test_config – load required data for tests
+    if [ "$CYPRESS_CONFIG" == "true" ]; then
+        superset load_test_users
+        superset load_examples --load-test-data
+    else
+        superset load_examples
+    fi
+    echo_step "4" "Complete" "Loading examples"
+fi
+echo_step "5" "Start" "Updating dashboards"
+superset import-directory /dashboards -f -o
+echo_step "5" "Complete" "Updating dashboards"
+echo_step "6" "Start" "Updating datasources"
+superset set_database_uri --database_name $ANALYTICS_DATASOURCE_NAME --uri postgresql://$ANALYTICS_DB_USER:$ANALYTICS_DB_PASSWORD@$ANALYTICS_DB_HOST:5432/$ANALYTICS_DB_NAME
+echo_step "6" "Complete" "Updating datasources"

docker/superset/config/superset_config.py

@@ -0,0 +1,129 @@
+import logging
+import os
+from dotenv import load_dotenv
+from cachelib import RedisCache
+
+from cachelib.file import FileSystemCache
+logger = logging.getLogger()
+
+def password_from_env(url):
+    return os.getenv("ANALYTICS_DB_PASSWORD")
+
+SQLALCHEMY_CUSTOM_PASSWORD_STORE = password_from_env
+
+def get_env_variable(var_name, default=None):
+    """Get the environment variable or raise exception."""
+    try:
+        return os.environ[var_name]
+    except KeyError:
+        if default is not None:
+            return default
+        else:
+            error_msg = "The environment variable {} was missing, abort...".format(
+                var_name
+            )
+            raise EnvironmentError(error_msg)
+
+
+MAPBOX_API_KEY = os.getenv('MAPBOX_API_KEY', '')
+
+DATABASE_DIALECT = get_env_variable("DATABASE_DIALECT", "postgres")
+DATABASE_USER = get_env_variable("DATABASE_USER", "superset")
+DATABASE_PASSWORD = get_env_variable("DATABASE_PASSWORD", "superset")
+DATABASE_HOST = get_env_variable("DATABASE_HOST", "postgres")
+DATABASE_PORT = get_env_variable("DATABASE_PORT", 5432)
+DATABASE_DB = get_env_variable("DATABASE_DB", "superset")
+
+SQLALCHEMY_TRACK_MODIFICATIONS = get_env_variable("SQLALCHEMY_TRACK_MODIFICATIONS", True)
+SECRET_KEY = get_env_variable("SECRET_KEY", 'thisISaSECRET_1234')
+
+# The SQLAlchemy connection string.
+SQLALCHEMY_DATABASE_URI = "%s://%s:%s@%s:%s/%s" % (
+    DATABASE_DIALECT,
+    DATABASE_USER,
+    DATABASE_PASSWORD,
+    DATABASE_HOST,
+    DATABASE_PORT,
+    DATABASE_DB,
+)
+
+REDIS_HOST = get_env_variable("REDIS_HOST", "redis")
+REDIS_PORT = get_env_variable("REDIS_PORT", 6379)
+REDIS_CELERY_DB = get_env_variable("REDIS_CELERY_DB", 0)
+REDIS_RESULTS_DB = get_env_variable("REDIS_CELERY_DB", 1)
+
+RESULTS_BACKEND = RedisCache(host=REDIS_HOST, port=REDIS_PORT, key_prefix='superset_results')
+# RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab")
+
+class CeleryConfig(object):
+    BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
+    CELERY_IMPORTS = ("superset.sql_lab",)
+    CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
+    CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}}
+    CELERY_TASK_PROTOCOL = 1
+
+CACHE_CONFIG = {
+    'CACHE_TYPE': 'redis',
+    'CACHE_DEFAULT_TIMEOUT': 300,
+    'CACHE_KEY_PREFIX': 'superset_',
+    'CACHE_REDIS_HOST': 'redis',
+    'CACHE_REDIS_PORT': 6379,
+    'CACHE_REDIS_DB': 1,
+    'CACHE_REDIS_URL': 'redis://redis:6379/1'
+}
+
+CELERY_CONFIG = CeleryConfig
+SQLLAB_CTAS_NO_LIMIT = True
+PERMANENT_SESSION_LIFETIME = 86400
+
+class ReverseProxied(object):
+
+    def __init__(self, app):
+        self.app = app
+
+    def __call__(self, environ, start_response):
+        script_name = environ.get('HTTP_X_SCRIPT_NAME', '')
+        if script_name:
+            environ['SCRIPT_NAME'] = script_name
+            path_info = environ['PATH_INFO']
+            if path_info.startswith(script_name):
+                environ['PATH_INFO'] = path_info[len(script_name):]
+
+        scheme = environ.get('HTTP_X_SCHEME', '')
+        if scheme:
+            environ['wsgi.url_scheme'] = scheme
+        return self.app(environ, start_response)
+
+
+ADDITIONAL_MIDDLEWARE = [ReverseProxied, ]
+ENABLE_PROXY_FIX = True
+
+# Enable the security manager API.
+FAB_ADD_SECURITY_API = True
+
+if os.getenv("ENABLE_OAUTH") == "true":
+    from flask_appbuilder.security.manager import AUTH_OAUTH
+    from security import CustomSecurityManager
+    AUTH_ROLES_SYNC_AT_LOGIN = True
+    AUTH_USER_REGISTRATION = True
+    AUTH_USER_REGISTRATION_ROLE = "Admin"
+    CUSTOM_SECURITY_MANAGER = CustomSecurityManager
+    LOGOUT_REDIRECT_URL = os.environ.get("SUPERSET_URL")
+    AUTH_TYPE = AUTH_OAUTH
+    OAUTH_PROVIDERS = [
+        {
+            'name': 'keycloak',
+            'token_key': 'access_token',  # Name of the token in the response of access_token_url
+            'icon': 'fa-key',   # Icon for the provider
+            'remote_app': {
+                'client_id': os.environ.get("SUPERSET_CLIENT_ID","superset"),  # Client Id (Identify Superset application)
+                'client_secret': os.environ.get("SUPERSET_CLIENT_SECRET"),  # Secret for this Client Id (Identify Superset application)
+                'api_base_url': os.environ.get("ISSUER_URL").rstrip('/') + "/protocol/openid-connect/",
+                'client_kwargs': {
+                    'scope': 'openid profile email',
+                },
+                'logout_redirect_uri': os.environ.get("SUPERSET_URL"),
+                'server_metadata_url': os.environ.get("ISSUER_URL").rstrip('/') + '/.well-known/openid-configuration',  # URL to get metadata from
+            }
+        }
+    ]

scripts/start-with-sso.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+source utils.sh
+
+export ENABLE_OAUTH=true
+echo "$INFO Setting ENABLE_OAUTH=true..."
+echo "→ ENABLE_OAUTH=$ENABLE_OAUTH"
+
+source start.sh

scripts/start.sh

@@ -15,6 +15,6 @@ setTraefikIP
 
 setTraefikHostnames
 
-echo $CONNECT_ODOO_DB_NAME
+echo "$CONNECT_ODOO_DB_NAME"
 
 docker compose -p ozone-analytics -f ../docker/docker-compose-db.yaml -f ../docker/docker-compose-migration.yaml -f ../docker/docker-compose-streaming-common.yaml -f ../docker/docker-compose-kowl.yaml  -f ../docker/docker-compose-superset.yaml -f ../docker/docker-compose-superset-ports.yaml up -d

scripts/utils.sh

@@ -49,7 +49,7 @@ function exportEnvs () {
     export ANALYTICS_QUERIES_PATH=$ANALYTICS_CONFIG_PATH/dsl/flattening/queries
     export ANALYTICS_DESTINATION_TABLES_MIGRATIONS_PATH=$ANALYTICS_CONFIG_PATH/liquibase/analytics
     export SQL_SCRIPTS_PATH=$DISTRO_PATH/data
-    export SUPERSET_CONFIG_PATH=$DISTRO_PATH/configs/superset/
+    export SUPERSET_CONFIG_PATH=../docker/superset/config
     export SUPERSET_DASHBOARDS_PATH=$DISTRO_PATH/configs/superset/assets/
     export JAVA_OPTS='-Xms2048m -Xmx8192m';
 
@@ -123,4 +123,4 @@ function setTraefikHostnames {
     export KEYCLOAK_HOSTNAME=auth-"${IP_WITH_DASHES}.traefik.me"
     echo "→ SUPERSET_HOSTNAME=$SUPERSET_HOSTNAME"
     echo "→ KEYCLOAK_HOSTNAME=$KEYCLOAK_HOSTNAME"
-}
+}