setup mother duck instead of local duckdb

Signed-off-by: Raphaël Courivaud <[email protected]>

analytics/dagster/src/assets/dwh/setup_duckdb.py

@@ -1,4 +1,4 @@
-from dagster import asset
+from dagster import MaterializeResult, asset
 from dagster_duckdb import DuckDBResource
 from ...config import Config
 
@@ -10,12 +10,18 @@
     compute_kind="duckdb",
 )
 def setup_duckdb(context, duckdb: DuckDBResource):
-    SETUP_QUERY = f"""
-    SET memory_limit = '{Config.DUCKDB_MEMORY_LIMIT}GB';
-    SET threads TO {Config.DUCKDB_THREAD_NUMBER};
-    """
+    context.log.info(f"Config.USE_MOTHER_DUCK{Config.USE_MOTHER_DUCK}")
 
+    if not Config.USE_MOTHER_DUCK:
+        SETUP_QUERY = f"""
+        SET memory_limit = '{Config.DUCKDB_MEMORY_LIMIT}GB';
+        SET threads TO {Config.DUCKDB_THREAD_NUMBER};
+        """
 
-    with duckdb.get_connection() as conn:
-        context.log.info(f"Executing SQL: {SETUP_QUERY}")
-        conn.execute(SETUP_QUERY)
+        with duckdb.get_connection() as conn:
+            context.log.info(f"Executing SQL: {SETUP_QUERY}")
+            conn.execute(SETUP_QUERY)
+
+        return "DuckDB setup successfully"
+    else:
+        return "Mother Duck is used, no need to setup DuckDB"

analytics/dagster/src/config.py

@@ -25,6 +25,11 @@ class Config:
     DUCKDB_MEMORY_LIMIT = os.environ.get("DUCKDB_MEMORY_LIMIT")
     DUCKDB_THREAD_NUMBER = os.environ.get("DUCKDB_THREAD_NUMBER", 4)
     METABASE_APP_ID = os.environ.get("METABASE_APP_ID")
+
+    MD_TOKEN = os.environ.get("MD_TOKEN")
+    USE_MOTHER_DUCK = os.environ.get("USE_MOTHER_DUCK", True)
+    USE_MOTHER_DUCK_FOR_METABASE = os.environ.get("USE_MOTHER_DUCK_FOR_METABASE", False)
+
 
 
 

analytics/dagster/src/definitions.py

@@ -8,7 +8,7 @@
 
 # from .assets import dagster_production_assets
 from .assets import dwh
-
+from .config import Config
 
 # from dagster_embedded_elt.dlt import DagsterDltResource
 from dagster_dbt import DbtCliResource
@@ -91,10 +91,10 @@
         # "dlt": dlt_resource,
         "dbt": dbt_resource,
         "duckdb": DuckDBResource(
-            database="db/dagster.duckdb",  # required
+            database=f"md:dwh?motherduck_token={Config.MD_TOKEN}" if Config.USE_MOTHER_DUCK else "db/dagster.duckdb",
         ),
         "duckdb_metabase": DuckDBResource(
-            database="db/duckdb_metabase.duckdb",  # required
+            database=f"md:metabase?motherduck_token={Config.MD_TOKEN} " if Config.USE_MOTHER_DUCK_FOR_METABASE else "db/metabase.duckdb",
         ),
     },
     schedules=[daily_refresh_schedule, yearly_ff_refresh_schedule],

analytics/dbt/profiles.yml

@@ -18,7 +18,7 @@ duckdb_profile:
 
     prod:
       type: duckdb
-      path: "/opt/dagster/dagster_home/db/dagster.duckdb"
+      path: "md:dwh?motherduck_token={{env_var('MD_TOKEN')}}"
       extensions:
         - httpfs
         - parquet