Version 1.0.9

PUBLISH

@@ -7,4 +7,7 @@ Build new dist: python setup.py sdist bdist_wheel
 
 Upload test: python -m twine upload --repository testpypi dist/*
 
-Upload live: python -m twine upload --skip-existing dist/*
+Upload live: python -m twine upload --skip-existing dist/*
+
+--
+python -m pip install -e C:\Sources\self\snowbim\dist

README.md

@@ -1,5 +1,5 @@
 # snowbim
-This is to do something awesome between snowflake database and tabular model file (.bim).
+This is to do something awesome between snowflake database and Power BI tabular model file (.bim).
 
 Those are:
 * Refresh tables (key: name)
@@ -10,7 +10,53 @@ Supported Models:
 * Compatibility Level: 1550
 * Default Power BI Data Source Version: powerBI_V3
 
+We make use of [dbt (data build tool)'s profile](https://docs.getdbt.com/dbt-cli/configure-your-profile) to configure the Snowflake connection.
+> Therefore, PLEASE MAKE SURE you've already managed to create 1 dbt profiles.yml file.
 
+Installation:
+```
+python -m pip install snowbim --upgrade
+
+# dependencies
+python -m pip install snowflake-connector-python[pandas]
+
+# check version
+python -m snowbim --version
+```
+
+
+## Usage
+```
+python -m snowbim --help
+```
+
+If you don't use --profile-dir argument, by default, it will look for profiles.yml in the user home folder:
+* Windows:  %userprofile%/.dbt/profiles.yml
+* Linux:    ~/.dbt/profiles.yml
+
+Sample commands:
+* To create new model.bim file:
+```
+python -m snowbim --bim "/path/to/model.bim" --db "YOUR_SF_DB_NAME" --schema "YOUR_SF_SCHEMA_NAME"
+# model.bim will be created after above command
+```
+
+* To upgrade existing model.bim file:
+```
+python -m snowbim --bim "/path/to/model.bim" --db "YOUR_SF_DB_NAME" --schema "YOUR_SF_SCHEMA_NAME"
+# model.bim will be overidden after above command
+```
+
+* To upgrade existing model.bim file but output to a new model_upgrade.bim file:
+```
+python -m snowbim --bim "/path/to/model.bim" --out "/path/to/model_upgrade.bim" --db "YOUR_SF_DB_NAME" --schema "YOUR_SF_SCHEMA_NAME"
+# model_upgrade.bim will be created after above command
+```
+
+> NOTE: If schema is up-to-date, .bim file will not be created or modified.
+
+
+## Development Enviroment
 Virtual enviroment:
 ```
 python -m venv env

setup.py

@@ -2,7 +2,7 @@
 
 setup(
      name='snowbim',
-     version='1.0.7',
+     version='1.0.9',
      author='datnguye',
      author_email='[email protected]',
      packages=find_packages(),
@@ -13,7 +13,8 @@
      long_description=open('README.md').read(),
      install_requires=[
         'pyyaml==5.4.1',
-        'snowflake-connector-python==2.4.3'
+        'snowflake-connector-python==2.4.3',
+        'snowflake-connector-python[pandas]'
      ],
      python_requires='>=3.7.5',
      entry_points = {

snowbim/__main__.py

@@ -2,24 +2,25 @@
 from snowbim import snowbim
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--bim',        help='File path to an input .bim file', type=str, default='')
-    parser.add_argument('--out',        help='File path to an out .bim file. Optional', type=str, default='')
-    parser.add_argument('--profile',    help='dbt profile name. Optional', type=str, default=None)
-    parser.add_argument('--target',     help='dbt profile target. Optional, default: dev', type=str, default='dev')
-    parser.add_argument('--db',         help='Snowflake database. Optional, default: DEMO_DB', type=str, default='DEMO_DB')
-    parser.add_argument('--schema',     help='Snowflake database\'s schema. Optional, default: PUBLIC', type=str, default='PUBLIC')
+    parser = argparse.ArgumentParser(prog='snowbim')
+    parser.add_argument('-v','--version', action='version', version='%(prog)s 1.0.9')
+
+    parser.add_argument('--bim',            help='File path to an input .bim file', type=str, default='')
+    parser.add_argument('--out',            help='File path to an out .bim file. Optional', type=str, default='')
+    parser.add_argument('--profile_dir',    help='File path to dbt profiles.yml. Optional, default to .dbt folder within User Home one', type=str, default=None)
+    parser.add_argument('--profile',        help='dbt profile name. Optional, default: Fisrt profile in profiles.yml', type=str, default=None)
+    parser.add_argument('--target',         help='dbt profile target. Optional, default: dev', type=str, default='dev')
+    parser.add_argument('--db',             help='Snowflake database. Optional, default: DEMO_DB', type=str, default='DEMO_DB')
+    parser.add_argument('--schema',         help='Snowflake database\'s schema. Optional, default: PUBLIC', type=str, default='PUBLIC')
 
     args = parser.parse_args()
     snowbim.upgrade_schema( bim_path        = args.bim, 
                             out_bim_path    = args.out,
+                            profile_dir     = args.profile_dir,
                             profile         = args.profile,
                             target          = args.target,
                             db              = args.db,
                             schema          = args.schema)
 
 if __name__ == '__main__':
-    main()
-
-# Samples:
-# snowbim --bim "C:\Users\PowerBITabularModel_upgrade.bim"
+    main()

snowbim/engines/snowengine.py

@@ -8,26 +8,30 @@
 from snowbim.utilities import common
 
 
-def connect(profile:str=None, project_name:str=None, target:str=None, db:str=None, schema:str=None):
+def connect(profile_dir:str=None, profile:str=None, target:str=None, db:str=None, schema:str=None):
     '''
     Use dbt profiles.yml to connect snowflake database
     Returns a tuple (code, snowflake connection, message)
     '''
     conn = {}
 
-    if profile is None:
-        profile = f'{expanduser("~")}/.dbt/profiles.yml'
+    profile_path = ''
+    if not profile_dir:
+        profile_path = f'{expanduser("~")}/.dbt/profiles.yml'
+    else:
+        profile_path = f'{profile_dir}/profiles.yml'
+    print(f'dbt profile was found at: {profile_path}')
 
-    with open(profile, 'r') as stream:
+    with open(profile_path, 'r') as stream:
         try:
             cred = yaml.load(stream, Loader=FullLoader)
         except yaml.YAMLError as e:
             return (-1, {}, str(e))
 
-    if project_name is None:
+    if profile is None:
         cred = cred[next(iter(cred))]
     else:
-        cred = cred[project_name]
+        cred = cred[profile]
 
     cred = cred['outputs']
     if target is None:
@@ -36,7 +40,7 @@ def connect(profile:str=None, project_name:str=None, target:str=None, db:str=Non
         cred = cred[target]
 
     if cred is None or cred['type'] != 'snowflake':
-        return (-1, {}, f'Snowflake config not found: project={str(project_name)}, target={str(target)} within profile: {profile}')
+        return (-1, {}, f'Snowflake config not found: project={str(profile)}, target={str(target)} within profile: {profile_path}')
 
     # cred = {'type': 'snowflake', 'account': 'xxx', 'user': 'xxx', 'password': 'xxx', 'role': 'xxx', 'database': 'xxx', 'warehouse': 'xxx', 'schema': 'xxx'}
     conn = sfconn.connect(

snowbim/snowbim.py

@@ -1,8 +1,11 @@
 from snowbim.engines import snowengine, bimengine
 
-def get_schema_changes(bim_path=None, profile:str=None, target:str=None, db:str=None, schema:str=None):
+def get_schema_changes(bim_path=None, profile_dir:str=None, profile:str=None, target:str=None, db:str=None, schema:str=None):
+    '''
+    get_schema_changes
+    '''
     print('Connecting to Snowflake...')
-    conn = snowengine.connect(profile=profile, target=target, db=db, schema=schema)
+    conn = snowengine.connect(profile_dir=profile_dir, profile=profile, target=target, db=db, schema=schema)
     print('     Connected')
 
     print('Schema comparing...')
@@ -11,8 +14,11 @@ def get_schema_changes(bim_path=None, profile:str=None, target:str=None, db:str=
 
     return changes
 
-def upgrade_schema(bim_path=None, out_bim_path=None, profile:str=None, target:str=None, db:str=None, schema:str=None):
-    changes = get_schema_changes(bim_path=bim_path, profile=profile, target=target, db=db, schema=schema)
+def upgrade_schema(bim_path=None, out_bim_path=None, profile_dir:str=None, profile:str=None, target:str=None, db:str=None, schema:str=None):
+    '''
+    upgrade_schema
+    '''
+    changes = get_schema_changes(bim_path=bim_path, profile_dir=profile_dir, profile=profile, target=target, db=db, schema=schema)
     if changes[0] == 0:
         changes = changes[1]
     else: