diff --git a/config/development.yaml b/config/development.yaml index 2149923..d63c2e3 100644 --- a/config/development.yaml +++ b/config/development.yaml @@ -14,5 +14,5 @@ validate: # HARE concept id: - '2000007027' catch_all_cohort_id: 4 -worker_pool_size: 5 -batch_size: 50 +worker_pool_size: 2 +batch_size: 4 diff --git a/models/datadictionary.go b/models/datadictionary.go index 492f50a..56766e7 100644 --- a/models/datadictionary.go +++ b/models/datadictionary.go @@ -75,8 +75,9 @@ func (u DataDictionary) GetDataDictionary() (*DataDictionaryModel, error) { } var dataSourceModel = new(Source) omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Omop) + miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Misc) - if u.CheckIfDataDictionaryIsFilled(omopDataSource) { + if u.CheckIfDataDictionaryIsFilled(miscDataSource) { var newDataDictionary DataDictionaryModel var dataDictionaryEntries []*DataDictionaryResult //Get total number of person ids @@ -95,7 +96,7 @@ func (u DataDictionary) GetDataDictionary() (*DataDictionaryModel, error) { } //get data dictionary entires saved in table - query = omopDataSource.Db.Table(omopDataSource.Schema + ".data_dictionary_result") + query = miscDataSource.Db.Table(miscDataSource.Schema + ".data_dictionary_result") query, cancel = utils.AddSpecificTimeoutToQuery(query, 600*time.Second) defer cancel() meta_result = query.Scan(&dataDictionaryEntries) @@ -110,7 +111,7 @@ func (u DataDictionary) GetDataDictionary() (*DataDictionaryModel, error) { newDataDictionary.Data, _ = json.Marshal(dataDictionaryEntries) //set in cache ResultCache = &newDataDictionary - return ResultCache, nil + return &newDataDictionary, nil } else { return nil, errors.New("data dictionary is not available yet") } @@ -138,8 +139,9 @@ func (u DataDictionary) GenerateDataDictionary() { } var dataSourceModel = new(Source) omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Omop) + miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, Misc) - if u.CheckIfDataDictionaryIsFilled(omopDataSource) { + if u.CheckIfDataDictionaryIsFilled(miscDataSource) { log.Print("Data Dictionary Result already filled. Skipping generation.") return } else { @@ -183,13 +185,13 @@ func (u DataDictionary) GenerateDataDictionary() { resultDataList = append(resultDataList, partialResultList...) if len(resultDataList) >= batchSize { log.Printf("%v row of results reached, flush to db.", batchSize) - u.WriteResultToDB(omopDataSource, resultDataList) + u.WriteResultToDB(miscDataSource, resultDataList) resultDataList = []*DataDictionaryResult{} } } if len(resultDataList) > 0 { - u.WriteResultToDB(omopDataSource, resultDataList) + u.WriteResultToDB(miscDataSource, resultDataList) } log.Printf("INFO: Data dictionary generation complete") diff --git a/models/source.go b/models/source.go index a3e7633..0a8d677 100644 --- a/models/source.go +++ b/models/source.go @@ -63,6 +63,7 @@ const ( Omop SourceType = 0 //TODO - we might have to split up into OmopData and OmopVocab in future... Results SourceType = 2 Temp SourceType = 5 + Misc SourceType = 6 ) // Get the data source details for given source id and source type. diff --git a/tests/models_tests/models_test.go b/tests/models_tests/models_test.go index 5c74bc4..4771573 100644 --- a/tests/models_tests/models_test.go +++ b/tests/models_tests/models_test.go @@ -1045,7 +1045,7 @@ func TestPersonConceptAndCountString(t *testing.T) { } -func TestGetDataDictionary(t *testing.T) { +func TestGetDataDictionaryFail(t *testing.T) { setUp(t) data, _ := dataDictionaryModel.GetDataDictionary() @@ -1060,14 +1060,14 @@ func TestCheckIfDataDictionaryIsFilled(t *testing.T) { var source = new(models.Source) sources, _ := source.GetAllSources() var dataSourceModel = new(models.Source) - omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Omop) + miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Misc) - filled := dataDictionaryModel.CheckIfDataDictionaryIsFilled(omopDataSource) + filled := dataDictionaryModel.CheckIfDataDictionaryIsFilled(miscDataSource) if filled != false { t.Errorf("Flag should be false") } dataDictionaryModel.GenerateDataDictionary() - filled = dataDictionaryModel.CheckIfDataDictionaryIsFilled(omopDataSource) + filled = dataDictionaryModel.CheckIfDataDictionaryIsFilled(miscDataSource) if filled != true { t.Errorf("Flag should be true") } @@ -1088,10 +1088,10 @@ func TestWriteToDB(t *testing.T) { var source = new(models.Source) sources, _ := source.GetAllSources() var dataSourceModel = new(models.Source) - omopDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Omop) + miscDataSource := dataSourceModel.GetDataSource(sources[0].SourceId, models.Misc) resultList := append([]*models.DataDictionaryResult{}, &models.DataDictionaryResult{ConceptID: 123}) - success := dataDictionaryModel.WriteResultToDB(omopDataSource, resultList) + success := dataDictionaryModel.WriteResultToDB(miscDataSource, resultList) //Write succeeded without panicking if success != true { t.Errorf("Write failed") diff --git a/tests/setup_local_db/ddl_results_and_cdm.sql b/tests/setup_local_db/ddl_results_and_cdm.sql index 2f992ab..4d78518 100644 --- a/tests/setup_local_db/ddl_results_and_cdm.sql +++ b/tests/setup_local_db/ddl_results_and_cdm.sql @@ -95,25 +95,6 @@ CREATE TABLE omop.concept invalid_reason character varying(1) COLLATE pg_catalog."default" ); -CREATE TABLE omop.DATA_DICTIONARY_RESULT -( - vocabulary_id character varying(20), - concept_id integer not null, - concept_code character varying(50), - concept_name character varying(255), - concept_class_id character varying(20), - number_of_people_with_variable integer, - number_of_people_where_value_is_filled integer, - number_of_people_where_value_is_null integer, - value_stored_as character varying(20), - min_value numeric, - max_value numeric, - mean_value numeric, - standard_deviation numeric, - value_summary JSON --For sql server use varchar(max) -); -ALTER TABLE omop.DATA_DICTIONARY_RESULT ADD CONSTRAINT xpk_DATA_DICTIONARY_RESULT PRIMARY KEY ( concept_id ) ; - CREATE VIEW omop.OBSERVATION_CONTINUOUS AS SELECT ob.person_id, ob.observation_concept_id, ob.value_as_string, ob.value_as_number, ob.value_as_concept_id FROM omop.observation ob @@ -157,3 +138,27 @@ GROUP BY c.vocabulary_id, c.concept_id, c.concept_code, c.concept_name, c.concep cc.number_of_people_where_value_is_filled_concept, cc.number_of_people_where_value_is_null_number, cc.number_of_people_where_value_is_null_concept; + +-- ======================================================== +DROP SCHEMA IF EXISTS misc CASCADE; +CREATE SCHEMA misc; +-- ======================================================== + +CREATE TABLE misc.DATA_DICTIONARY_RESULT +( + vocabulary_id character varying(20), + concept_id integer not null, + concept_code character varying(50), + concept_name character varying(255), + concept_class_id character varying(20), + number_of_people_with_variable integer, + number_of_people_where_value_is_filled integer, + number_of_people_where_value_is_null integer, + value_stored_as character varying(20), + min_value numeric, + max_value numeric, + mean_value numeric, + standard_deviation numeric, + value_summary JSON --For sql server use varbinary(max) +); +ALTER TABLE misc.DATA_DICTIONARY_RESULT ADD CONSTRAINT xpk_DATA_DICTIONARY_RESULT PRIMARY KEY ( concept_id ) ; diff --git a/tests/setup_local_db/test_data_atlas.sql b/tests/setup_local_db/test_data_atlas.sql index 5c936e8..2abc108 100644 --- a/tests/setup_local_db/test_data_atlas.sql +++ b/tests/setup_local_db/test_data_atlas.sql @@ -14,7 +14,8 @@ values (1,1,0, 'OMOP', 1), (2,1,1, 'OMOP', 1), (3,1,2, 'RESULTS', 1), - (4,1,5, 'TEMP', 1) + (4,1,5, 'TEMP', 1), + (5,1,6, 'MISC', 1) ; insert into atlas.cohort_definition