self review cleanup

cumulus_library/studies/core/builder_observation.py

@@ -30,8 +30,8 @@
     }
 }
 
-# TODO: upgrade to 3.10+, use kw_only flag to subclass for generating source/target
 
+# TODO: upgrade to 3.10+, use kw_only flag to subclass a dataclass for generating source/target
 code_sources = [
     CCC(column_name="category", is_array=True, filter_priority=False),
     CCC(column_name="code", is_array=False, filter_priority=False),
@@ -66,4 +66,3 @@ def prepare_queries(
         self.queries.append(
             core_templates.get_core_template("observation", validated_schema)
         )
-        self.write_queries()

cumulus_library/studies/core/builder_patient.py

@@ -1,9 +1,6 @@
 """ Module for extracting US core extensions from patient records"""
 from cumulus_library.base_table_builder import BaseTableBuilder
-from cumulus_library.template_sql.templates import (
-    get_extension_denormalize_query,
-    ExtensionConfig,
-)
+from cumulus_library.template_sql import templates, utils
 from cumulus_library import databases
 from cumulus_library.studies.core.core_templates import core_templates
 
@@ -46,7 +43,7 @@ def prepare_queries(
         ]
 
         for extension in extension_types:
-            config = ExtensionConfig(
+            config = utils.ExtensionConfig(
                 "patient",
                 "id",
                 f"core__patient_ext_{extension['name']}",
@@ -55,7 +52,7 @@ def prepare_queries(
                 ["ombCategory", "detailed", "text"],
                 is_array=True,
             )
-            self.queries.append(get_extension_denormalize_query(config))
+            self.queries.append(templates.get_extension_denormalize_query(config))
         validated_schema = core_templates.validate_schema(
             cursor, schema, expected_table_cols, parser
         )

cumulus_library/studies/core/core_templates/documentreference.sql.jinja

@@ -23,7 +23,19 @@ WITH temp_documentreference AS (
                 ], 
                 schema
             ) 
-        }}
+        }},
+        {{- utils.truncate_date_cols(
+                'documentreference',
+                'dr',
+                [
+                    ('author_date', 'day'),
+                    ('author_date', 'week'),
+                    ('author_date', 'month'),
+                    ('author_date', 'year'),
+                ],
+                schema
+            ) 
+        }},
     FROM documentreference AS dr
 )
 
@@ -39,14 +51,14 @@ SELECT DISTINCT
     tdr.status,
     tdr.docstatus,
     context_encounter.encounter.reference AS encounter_ref,
-    date_trunc('day', date(from_iso8601_timestamp(tdr.author_date))) AS author_date,
-    date_trunc('week', date(from_iso8601_timestamp(tdr.author_date))) AS author_week,
-    date_trunc('month', date(from_iso8601_timestamp(tdr.author_date))) AS author_month,
-    date_trunc('year', date(from_iso8601_timestamp(tdr.author_date))) AS author_year,
+    author_day AS author_date,
+    author_week,
+    author_month,
+    author_year,
     tdr.subject_ref,
     tdr.id as doc_id,
     concat('DocumentReference/', tdr.id) AS doc_ref
 FROM temp_documentreference AS tdr,
     unnest(context.encounter) AS context_encounter (encounter), --noqa
     unnest(type.coding) AS type_coding (type_row)
-WHERE date(from_iso8601_timestamp(tdr.author_date)) BETWEEN date('2016-06-01') AND current_date;
+WHERE author_day BETWEEN date('2016-06-01') AND current_date;

cumulus_library/study_parser.py

@@ -637,7 +637,9 @@ def export_study(
             dataframe = dataframe.sort_values(
                 by=list(dataframe.columns), ascending=False, na_position="first"
             )
-            dataframe.to_csv(f"{path}/{table}.csv", index=False, quoting=csv.QUOTE_ALL)
+            dataframe.to_csv(
+                f"{path}/{table}.csv", index=False, quoting=csv.QUOTE_MINIMAL
+            )
             dataframe.to_parquet(f"{path}/{table}.parquet", index=False)
             queries.append(query)
         return queries

cumulus_library/template_sql/statistics/counts_templates.py

@@ -24,7 +24,7 @@ class CountableFhirResource(Enum):
     OBSERVATION = "observation"
     PATIENT = "patient"
     MEDICATION = "medication"
-    MEDICATIONQUEST = "medicationrequest"
+    MEDICATIONREQUEST = "medicationrequest"
 
 
 @dataclass

cumulus_library/template_sql/templates.py

@@ -5,7 +5,9 @@
 
 from jinja2 import Template
 from pandas import DataFrame
+
 from cumulus_library import databases
+from cumulus_library.template_sql import utils
 
 PATH = Path(__file__).parent
 
@@ -17,75 +19,6 @@ class TableView(Enum):
     VIEW = "VIEW"
 
 
-class CodeableConceptConfig:
-    """Convenience class for holding parameters for generating codableconcept tables.
-
-    :param source_table: the table to extract extensions from
-    :param source_id: the id field to use in the new table
-    :param column_name: the column containing the codeableConcept you want to extract.
-    :param is_array: whether the codeableConcept is 0...1 or 0..* in the FHIR spec
-    :param target_table: the name of the table to create
-    :param filter_priority: If true, will use code systems to select a single code,
-      in preference order, for use as a display value.
-    :param code_systems: a list of systems, in preference order, for selecting data
-      for filtering. This should not be set if filter_priority is false.
-    """
-
-    def __init__(
-        self,
-        source_table: str,
-        column_name: str,
-        is_array: bool,
-        target_table: str,
-        source_id: str = "id",
-        filter_priority: Optional[bool] = False,
-        code_systems: Optional[list] = None,
-    ):
-        if not filter_priority and code_systems != None:
-            raise Exception(
-                "CodeableConceptConfig cannot have non-default value assigned to "
-                "code_systems unless filter_priority is true."
-            )
-        self.source_table = source_table
-        self.source_id = source_id
-        self.column_name = column_name
-        self.is_array = is_array
-        self.target_table = target_table
-        self.filter_priority = filter_priority
-        self.code_systems = code_systems
-
-
-class ExtensionConfig(object):
-    """convenience class for holding parameters for generating extension tables.
-
-    :param source_table: the table to extract extensions from
-    :param source_id: the id column to treat as a foreign key
-    :param target_table: the name of the table to create
-    :param target_col_prefix: the string to prepend code/display column names with
-    :param fhir_extension: the URL of the FHIR resource to select
-    :param code_systems: a list of codes, in preference order, to use to select data
-    :param is_array: a boolean indicating if the targeted field is an array type
-    """
-
-    def __init__(
-        self,
-        source_table: str,
-        source_id: str,
-        target_table: str,
-        target_col_prefix: str,
-        fhir_extension: str,
-        ext_systems: List[str],
-        is_array: bool = False,
-    ):
-        self.source_table = source_table
-        self.source_id = source_id
-        self.target_table = target_table
-        self.target_col_prefix = target_col_prefix
-        self.fhir_extension = fhir_extension
-        self.ext_systems = ext_systems
-        self.is_array = is_array
-
-
 # TODO: Consolidate to a generic template reader
 
 
@@ -105,7 +38,7 @@ def get_code_system_pairs(output_table_name: str, code_system_tables: list) -> s
         )
 
 
-def get_codeable_concept_denormalize_query(config: CodeableConceptConfig) -> str:
+def get_codeable_concept_denormalize_query(config: utils.CodeableConceptConfig) -> str:
     """extracts codeable concepts from a specified table.
 
     This function is targeted at arbitrary codeableConcept elements - see
@@ -250,7 +183,7 @@ def get_drop_view_table(name: str, view_or_table: str) -> str:
             )
 
 
-def get_extension_denormalize_query(config: ExtensionConfig) -> str:
+def get_extension_denormalize_query(config: utils.ExtensionConfig) -> str:
     """extracts target extension from a table into a denormalized table
 
     This function is targeted at a complex extension element that is at the root

cumulus_library/template_sql/utils.py

@@ -42,10 +42,32 @@ class CodeableConceptConfig:
     code_systems: list = None
 
 
+@dataclass
+class ExtensionConfig(object):
+    """convenience class for holding parameters for generating extension tables.
+
+    :param source_table: the table to extract extensions from
+    :param source_id: the id column to treat as a foreign key
+    :param target_table: the name of the table to create
+    :param target_col_prefix: the string to prepend code/display column names with
+    :param fhir_extension: the URL of the FHIR resource to select
+    :param code_systems: a list of codes, in preference order, to use to select data
+    :param is_array: a boolean indicating if the targeted field is an array type
+    """
+
+    source_table: str
+    source_id: str
+    target_table: str
+    target_col_prefix: str
+    fhir_extension: str
+    ext_systems: List[str]
+    is_array: bool = False
+
+
 def _check_data_in_fields(
     schema,
     cursor,
-    code_sources: List[templates.CodeableConceptConfig],
+    code_sources: List[CodeableConceptConfig],
 ) -> dict:
     """checks if CodeableConcept fields actually have data available
 
@@ -89,7 +111,7 @@ def _check_data_in_fields(
 def denormalize_codes(
     schema: str,
     cursor: databases.DatabaseCursor,
-    code_sources: List[templates.CodeableConceptConfig],
+    code_sources: List[CodeableConceptConfig],
 ):
     queries = []
     code_sources = _check_data_in_fields(schema, cursor, code_sources)