Merge pull request #102 from uc-cdis/feat/vadc-1108

Change data dictionary result table schema to misc

config/development.yaml

@@ -14,5 +14,5 @@ validate:
     # HARE concept id:
     - '2000007027'
 catch_all_cohort_id: 4
-worker_pool_size: 5
-batch_size: 50
+worker_pool_size: 2
+batch_size: 4

models/datadictionary.go

@@ -75,8 +75,9 @@ func (u DataDictionary) GetDataDictionary() (*DataDictionaryModel, error) {
 		}
 		var dataSourceModel = new(Source)
 		omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Omop)
+		miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Misc)
 
-		if u.CheckIfDataDictionaryIsFilled(omopDataSource) {
+		if u.CheckIfDataDictionaryIsFilled(miscDataSource) {
 			var newDataDictionary DataDictionaryModel
 			var dataDictionaryEntries []*DataDictionaryResult
 			//Get total number of person ids
@@ -95,7 +96,7 @@ func (u DataDictionary) GetDataDictionary() (*DataDictionaryModel, error) {
 			}
 
 			//get data dictionary entires saved in table
-			query = omopDataSource.Db.Table(omopDataSource.Schema + ".data_dictionary_result")
+			query = miscDataSource.Db.Table(miscDataSource.Schema + ".data_dictionary_result")
 			query, cancel = utils.AddSpecificTimeoutToQuery(query, 600*time.Second)
 			defer cancel()
 			meta_result = query.Scan(&dataDictionaryEntries)
@@ -110,7 +111,7 @@ func (u DataDictionary) GetDataDictionary() (*DataDictionaryModel, error) {
 			newDataDictionary.Data, _ = json.Marshal(dataDictionaryEntries)
 			//set in cache
 			ResultCache = &newDataDictionary
-			return ResultCache, nil
+			return &newDataDictionary, nil
 		} else {
 			return nil, errors.New("data dictionary is not available yet")
 		}
@@ -138,8 +139,9 @@ func (u DataDictionary) GenerateDataDictionary() {
 	}
 	var dataSourceModel = new(Source)
 	omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Omop)
+	miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Misc)
 
-	if u.CheckIfDataDictionaryIsFilled(omopDataSource) {
+	if u.CheckIfDataDictionaryIsFilled(miscDataSource) {
 		log.Print("Data Dictionary Result already filled. Skipping generation.")
 		return
 	} else {
@@ -183,13 +185,13 @@ func (u DataDictionary) GenerateDataDictionary() {
 			resultDataList = append(resultDataList, partialResultList...)
 			if len(resultDataList) >= batchSize {
 				log.Printf("%v row of results reached, flush to db.", batchSize)
-				u.WriteResultToDB(omopDataSource, resultDataList)
+				u.WriteResultToDB(miscDataSource, resultDataList)
 				resultDataList = []*DataDictionaryResult{}
 			}
 		}
 
 		if len(resultDataList) > 0 {
-			u.WriteResultToDB(omopDataSource, resultDataList)
+			u.WriteResultToDB(miscDataSource, resultDataList)
 		}
 
 		log.Printf("INFO: Data dictionary generation complete")

models/source.go

@@ -63,6 +63,7 @@ const (
 	Omop    SourceType = 0 //TODO - we might have to split up into OmopData and OmopVocab in future...
 	Results SourceType = 2
 	Temp    SourceType = 5
+	Misc    SourceType = 6
 )
 
 // Get the data source details for given source id and source type.

tests/models_tests/models_test.go

@@ -1045,7 +1045,7 @@ func TestPersonConceptAndCountString(t *testing.T) {
 
 }
 
-func TestGetDataDictionary(t *testing.T) {
+func TestGetDataDictionaryFail(t *testing.T) {
 	setUp(t)
 
 	data, _ := dataDictionaryModel.GetDataDictionary()
@@ -1060,14 +1060,14 @@ func TestCheckIfDataDictionaryIsFilled(t *testing.T) {
 	var source = new(models.Source)
 	sources, _ := source.GetAllSources()
 	var dataSourceModel = new(models.Source)
-	omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Omop)
+	miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Misc)
 
-	filled := dataDictionaryModel.CheckIfDataDictionaryIsFilled(omopDataSource)
+	filled := dataDictionaryModel.CheckIfDataDictionaryIsFilled(miscDataSource)
 	if filled != false {
 		t.Errorf("Flag should be false")
 	}
 	dataDictionaryModel.GenerateDataDictionary()
-	filled = dataDictionaryModel.CheckIfDataDictionaryIsFilled(omopDataSource)
+	filled = dataDictionaryModel.CheckIfDataDictionaryIsFilled(miscDataSource)
 	if filled != true {
 		t.Errorf("Flag should be true")
 	}
@@ -1088,10 +1088,10 @@ func TestWriteToDB(t *testing.T) {
 	var source = new(models.Source)
 	sources, _ := source.GetAllSources()
 	var dataSourceModel = new(models.Source)
-	omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Omop)
+	miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Misc)
 
 	resultList := append([]*models.DataDictionaryResult{}, &models.DataDictionaryResult{ConceptID: 123})
-	success := dataDictionaryModel.WriteResultToDB(omopDataSource, resultList)
+	success := dataDictionaryModel.WriteResultToDB(miscDataSource, resultList)
 	//Write succeeded without panicking
 	if success != true {
 		t.Errorf("Write failed")

tests/setup_local_db/ddl_results_and_cdm.sql

@@ -95,25 +95,6 @@ CREATE TABLE omop.concept
     invalid_reason character varying(1) COLLATE pg_catalog."default"
 );
 
-CREATE TABLE omop.DATA_DICTIONARY_RESULT
-(
-    vocabulary_id character varying(20),
-    concept_id integer not null,
-    concept_code character varying(50),
-    concept_name character varying(255),
-    concept_class_id character varying(20),
-    number_of_people_with_variable integer,
-    number_of_people_where_value_is_filled integer,
-    number_of_people_where_value_is_null integer,
-    value_stored_as character varying(20),
-    min_value numeric,
-    max_value numeric,
-    mean_value numeric,
-    standard_deviation numeric,
-    value_summary JSON --For sql server use varchar(max)
-);
-ALTER TABLE omop.DATA_DICTIONARY_RESULT  ADD CONSTRAINT xpk_DATA_DICTIONARY_RESULT PRIMARY KEY ( concept_id ) ;
-
 CREATE VIEW omop.OBSERVATION_CONTINUOUS AS
 SELECT ob.person_id, ob.observation_concept_id, ob.value_as_string, ob.value_as_number, ob.value_as_concept_id
 FROM omop.observation ob
@@ -157,3 +138,27 @@ GROUP BY c.vocabulary_id, c.concept_id, c.concept_code, c.concept_name, c.concep
          cc.number_of_people_where_value_is_filled_concept,
          cc.number_of_people_where_value_is_null_number,
          cc.number_of_people_where_value_is_null_concept;
+
+-- ========================================================
+DROP SCHEMA IF EXISTS misc CASCADE;
+CREATE SCHEMA misc;
+-- ========================================================
+
+CREATE TABLE misc.DATA_DICTIONARY_RESULT
+(
+    vocabulary_id character varying(20),
+    concept_id integer not null,
+    concept_code character varying(50),
+    concept_name character varying(255),
+    concept_class_id character varying(20),
+    number_of_people_with_variable integer,
+    number_of_people_where_value_is_filled integer,
+    number_of_people_where_value_is_null integer,
+    value_stored_as character varying(20),
+    min_value numeric,
+    max_value numeric,
+    mean_value numeric,
+    standard_deviation numeric,
+    value_summary JSON --For sql server use varbinary(max)
+);
+ALTER TABLE misc.DATA_DICTIONARY_RESULT  ADD CONSTRAINT xpk_DATA_DICTIONARY_RESULT PRIMARY KEY ( concept_id ) ;

tests/setup_local_db/test_data_atlas.sql

@@ -14,7 +14,8 @@ values
     (1,1,0, 'OMOP', 1),
     (2,1,1, 'OMOP', 1),
     (3,1,2, 'RESULTS', 1),
-    (4,1,5, 'TEMP', 1)
+    (4,1,5, 'TEMP', 1),
+    (5,1,6, 'MISC', 1)
 ;
 
 insert into atlas.cohort_definition