From f69a4324002febc2efe5f7edf9347188b9a330ab Mon Sep 17 00:00:00 2001 From: Oren Date: Wed, 28 Aug 2024 18:23:40 +0300 Subject: [PATCH 1/5] added generate to the command line its hard coded to the folder - need to be fixed --- core_lib/core_lib_main.py | 27 ++++++++++++++++++- .../core_lib_config_generate_yaml.py | 4 +-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/core_lib/core_lib_main.py b/core_lib/core_lib_main.py index 30c173a5..27124990 100644 --- a/core_lib/core_lib_main.py +++ b/core_lib/core_lib_main.py @@ -2,10 +2,13 @@ import logging import os from core_lib.alembic.alembic import Alembic +import hydra from hydra import compose, initialize import click from dotenv import load_dotenv from core_lib.helpers.validation import is_int +from core_lib_generator.core_lib_config_generate_yaml import generate_core_lib_yaml +from core_lib_generator.core_lib_generator_from_yaml import CoreLibGenerator def list_to_string(lst: list): @@ -36,6 +39,28 @@ def load_config(): def main(): pass +@click.command() +@click.option('--yaml', help='name of new revision') +def generate(yaml): + if yaml: + if not os.path.exists(yaml): + click.echo(f"yaml file was not found\nmake sure you have the right path\npath:{yaml}\nexiting now...") + exit() + else: + yaml_file = generate_core_lib_yaml() + yaml = f'{os.getcwd()}/{yaml_file}' + + file_name = os.path.basename(yaml) + relative_path = os.path.relpath(yaml, os.getcwd()) + if file_name == relative_path: + relative_path = "." + relative_path = "..\\..\\..\\newfolder\\" + relative_path # TODO: change to upper folder instead of newfolder + print(f"relative_path={relative_path} | file name:{file_name} | cwd:{os.getcwd()}") + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize(config_path="..\\..\\..\\newfolder\\", caller_stack_depth=1, version_base='1.1') + config = hydra.compose(file_name) + CoreLibGenerator(config).run_all() + @click.command() @click.option('--rev', required=True, help=' '.join(get_rev_options())) @@ -78,5 +103,5 @@ def migrate(rev, name, env_file): # CoreLibGenerate().generate(list_to_string(value)) # # main.add_command(create) -# main.add_command(generate) +main.add_command(generate) main.add_command(migrate) diff --git a/core_lib_generator/core_lib_config_generate_yaml.py b/core_lib_generator/core_lib_config_generate_yaml.py index a5ce5dca..3cd8dd60 100644 --- a/core_lib_generator/core_lib_config_generate_yaml.py +++ b/core_lib_generator/core_lib_config_generate_yaml.py @@ -103,7 +103,7 @@ def create_yaml_file(core_lib_name: str): OmegaConf.save(config=conf, f=file.name) -def get_data_from_user(): +def generate_core_lib_yaml(): core_lib_name = any_to_pascal(input_str('Please enter the name for your Core-lib', 'MyCoreLib')) want_cache = input_yes_no('\nWould you like to use cache?', True) @@ -125,4 +125,4 @@ def get_data_from_user(): if __name__ == '__main__': - get_data_from_user() + generate_core_lib_yaml() From a569db6d83ff440f46ba0750ea1c4443e695def2 Mon Sep 17 00:00:00 2001 From: "shay.te@gmail.com" Date: Thu, 29 Aug 2024 10:15:59 +0300 Subject: [PATCH 2/5] update --- core_lib/core_lib_main.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/core_lib/core_lib_main.py b/core_lib/core_lib_main.py index 27124990..bc32c0df 100644 --- a/core_lib/core_lib_main.py +++ b/core_lib/core_lib_main.py @@ -3,7 +3,7 @@ import os from core_lib.alembic.alembic import Alembic import hydra -from hydra import compose, initialize +from hydra import compose, initialize, initialize_config_dir import click from dotenv import load_dotenv from core_lib.helpers.validation import is_int @@ -51,15 +51,20 @@ def generate(yaml): yaml = f'{os.getcwd()}/{yaml_file}' file_name = os.path.basename(yaml) + absolute_folter_path = os.path.dirname(yaml) relative_path = os.path.relpath(yaml, os.getcwd()) if file_name == relative_path: - relative_path = "." - relative_path = "..\\..\\..\\newfolder\\" + relative_path # TODO: change to upper folder instead of newfolder - print(f"relative_path={relative_path} | file name:{file_name} | cwd:{os.getcwd()}") + relative_path = "./" + # CHECK AND DELETE >>> + DEV_PATH = '' # '../../../' + relative_path = DEV_PATH + relative_path # TODO: change to upper folder instead of newfolder + # <<<<< hydra.core.global_hydra.GlobalHydra.instance().clear() - hydra.initialize(config_path="..\\..\\..\\newfolder\\", caller_stack_depth=1, version_base='1.1') + initialize_config_dir(config_dir=absolute_folter_path) config = hydra.compose(file_name) - CoreLibGenerator(config).run_all() + print(config) + # CoreLibGenerator(config).run_all() + @click.command() From 3b6fecb632ca462233b8a59f71d11d8fb32694aa Mon Sep 17 00:00:00 2001 From: Oren Date: Thu, 29 Aug 2024 16:32:07 +0300 Subject: [PATCH 3/5] added type to sql on yaml generate (added using test so also a meaningless test) --- core_lib/core_lib_main.py | 15 ++++----------- core_lib_generator/config_collectors/database.py | 5 +++++ tests/test_generate_data.py | 4 ++++ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/core_lib/core_lib_main.py b/core_lib/core_lib_main.py index bc32c0df..99a6546a 100644 --- a/core_lib/core_lib_main.py +++ b/core_lib/core_lib_main.py @@ -44,7 +44,7 @@ def main(): def generate(yaml): if yaml: if not os.path.exists(yaml): - click.echo(f"yaml file was not found\nmake sure you have the right path\npath:{yaml}\nexiting now...") + click.echo(f"yaml file was not found\nmake sure you have the right path (including .yaml or other extenions)\npath:{yaml}\nexiting now...") exit() else: yaml_file = generate_core_lib_yaml() @@ -52,19 +52,12 @@ def generate(yaml): file_name = os.path.basename(yaml) absolute_folter_path = os.path.dirname(yaml) - relative_path = os.path.relpath(yaml, os.getcwd()) - if file_name == relative_path: - relative_path = "./" - # CHECK AND DELETE >>> - DEV_PATH = '' # '../../../' - relative_path = DEV_PATH + relative_path # TODO: change to upper folder instead of newfolder - # <<<<< hydra.core.global_hydra.GlobalHydra.instance().clear() initialize_config_dir(config_dir=absolute_folter_path) config = hydra.compose(file_name) - print(config) - # CoreLibGenerator(config).run_all() - + CoreLibGenerator(config).run_all() + # copy required yaml file + # check how to change to actual yaml to work @click.command() diff --git a/core_lib_generator/config_collectors/database.py b/core_lib_generator/config_collectors/database.py index 58317613..71e31d28 100644 --- a/core_lib_generator/config_collectors/database.py +++ b/core_lib_generator/config_collectors/database.py @@ -85,9 +85,14 @@ def _generate_db_config( migrate: bool = False ) -> dict: config = _build_url(db_type, db_name, db_username, db_password, db_port, db_host) + if db_type: + db_type: str = 'core_lib.connection' + if migrate: + db_type += '.sql_alchemy_connection_registry.SqlAlchemyConnectionRegistry' return { 'env': config['env'], 'connection': { + 'type': db_type, 'key': db_name, 'migrate': migrate, 'log_queries': db_log_queries, diff --git a/tests/test_generate_data.py b/tests/test_generate_data.py index 1cb7da35..e3de9d7d 100644 --- a/tests/test_generate_data.py +++ b/tests/test_generate_data.py @@ -20,6 +20,10 @@ def has_special_chars(input_string: str): class TestGenerateData(unittest.TestCase): + def test_generate_yaml_connection(self): + from core_lib_generator.core_lib_config_generate_yaml import _get_data_layers_config + _get_data_layers_config() + def test_generate_string(self): string = generate_random_string() self.assertEqual(10, len(string)) From 8c150d61a3caa23f1f4872a51cc5157b4ecd5ecf Mon Sep 17 00:00:00 2001 From: Oren Date: Thu, 29 Aug 2024 16:43:16 +0300 Subject: [PATCH 4/5] removed comments --- core_lib/core_lib_main.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/core_lib/core_lib_main.py b/core_lib/core_lib_main.py index 99a6546a..d9f564d5 100644 --- a/core_lib/core_lib_main.py +++ b/core_lib/core_lib_main.py @@ -56,9 +56,6 @@ def generate(yaml): initialize_config_dir(config_dir=absolute_folter_path) config = hydra.compose(file_name) CoreLibGenerator(config).run_all() - # copy required yaml file - # check how to change to actual yaml to work - @click.command() @click.option('--rev', required=True, help=' '.join(get_rev_options())) From cc3037d368ce82b77f4cd3efa1a4592cc9b1d165 Mon Sep 17 00:00:00 2001 From: Oren Date: Mon, 2 Sep 2024 09:26:08 +0300 Subject: [PATCH 5/5] every option gets sql alchemy --- core_lib_generator/config_collectors/database.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core_lib_generator/config_collectors/database.py b/core_lib_generator/config_collectors/database.py index 71e31d28..5888eeb4 100644 --- a/core_lib_generator/config_collectors/database.py +++ b/core_lib_generator/config_collectors/database.py @@ -86,9 +86,7 @@ def _generate_db_config( ) -> dict: config = _build_url(db_type, db_name, db_username, db_password, db_port, db_host) if db_type: - db_type: str = 'core_lib.connection' - if migrate: - db_type += '.sql_alchemy_connection_registry.SqlAlchemyConnectionRegistry' + db_type: str = 'core_lib.connection.sql_alchemy_connection_registry.SqlAlchemyConnectionRegistry' return { 'env': config['env'], 'connection': {