diff --git a/python/idsse_common/idsse/common/schema/criteria_schema.json b/python/idsse_common/idsse/common/schema/criteria_schema.json index aa9e6cdf..f9528174 100644 --- a/python/idsse_common/idsse/common/schema/criteria_schema.json +++ b/python/idsse_common/idsse/common/schema/criteria_schema.json @@ -1,5 +1,5 @@ { - "description": "Mechanism for making communicating a Criteria", + "description": "Mechanism for communicating a Criteria", "allOf": [ {"$ref": "criteria.json#/Criteria"} ] diff --git a/python/idsse_common/idsse/common/schema/das_web_request.json b/python/idsse_common/idsse/common/schema/das_web_request.json new file mode 100644 index 00000000..d0e76952 --- /dev/null +++ b/python/idsse_common/idsse/common/schema/das_web_request.json @@ -0,0 +1,36 @@ +{ + "DasWebRequest": { + "description": "Mechanism for defining DAS data request via the web api", + "type": "object", + "properties": { + "issueDt": {"$ref": "timing.json#/TimeString"}, + "valids": {"$ref": "timing.json#/TimeList"}, + "dataRequest": { + "type": "string" + }, + "parts": { + "type": "array", + "items": {"$ref": "translated_data.json#/SingleTranslatedData"}, + "minItems": 1 + }, + "bbox": { + "type": "object", + "properties": { + "botLeft": {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 2}, + "topRight": {"type": "array", "items": {"type": "integer"}, "minItems": 2, "maxItems": 2} + }, + "required": [ + "botLeft", + "topRight" + ] + } + }, + "required": [ + "issueDt", + "valids", + "dataRequest", + "parts", + "bbox" + ] + } +} \ No newline at end of file diff --git a/python/idsse_common/idsse/common/schema/das_web_request_schema.json b/python/idsse_common/idsse/common/schema/das_web_request_schema.json new file mode 100644 index 00000000..6932afd5 --- /dev/null +++ b/python/idsse_common/idsse/common/schema/das_web_request_schema.json @@ -0,0 +1,6 @@ +{ + "description": "Mechanism for making DAS web request", + "allOf": [ + {"$ref": "das_web_request.json#/DasWebRequest"} + ] +} \ No newline at end of file diff --git a/python/idsse_common/idsse/common/schema/das_web_response_schema.json b/python/idsse_common/idsse/common/schema/das_web_response_schema.json new file mode 100644 index 00000000..5c0f9d98 --- /dev/null +++ b/python/idsse_common/idsse/common/schema/das_web_response_schema.json @@ -0,0 +1,29 @@ +{ + "description": "Mechanism for making DAS web request", + "allOf": [ + {"$ref": "das_web_request.json#/DasWebRequest"} + ], + "properties": { + "data": { + "description": "The keys in data are either a string datetime, 'scale', or 'offset' (nothing else allowed)", + "type": "object", + "patternProperties": { + "^(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}(?:\\.\\d*)?)((-(\\d{2}):(\\d{2})|Z)?)$": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "scale": {"type": "number"}, + "offset": {"type": "number"}, + "(?=^((?!((^|, )(scale|offset))+$).)*$)(?=[^0-9.+-:TZ])": {"not": {}} + } + } + }, + "required": [ + "data" + ] +} \ No newline at end of file diff --git a/python/idsse_common/idsse/common/schema/event_portfolio_schema.json b/python/idsse_common/idsse/common/schema/event_portfolio_schema.json index 25869d81..3367428c 100644 --- a/python/idsse_common/idsse/common/schema/event_portfolio_schema.json +++ b/python/idsse_common/idsse/common/schema/event_portfolio_schema.json @@ -9,5 +9,8 @@ "items": {"$ref": "risk_results.json#/RiskResults"}, "minItems": 1 } - } + }, + "required": [ + "riskResults" + ] } \ No newline at end of file diff --git a/python/idsse_common/idsse/common/schema/field.json b/python/idsse_common/idsse/common/schema/field.json index d541180a..ff386e1f 100644 --- a/python/idsse_common/idsse/common/schema/field.json +++ b/python/idsse_common/idsse/common/schema/field.json @@ -8,8 +8,7 @@ "FieldList": { "description": "A list of specific field(s)", "type": "array", - "items": {"$ref": "#/FieldName"}, - "minItems": 1 + "items": {"$ref": "#/FieldName"} }, "Field": { diff --git a/python/idsse_common/idsse/common/schema/new_data.json b/python/idsse_common/idsse/common/schema/new_data.json index 9dab5a1a..d26cf398 100644 --- a/python/idsse_common/idsse/common/schema/new_data.json +++ b/python/idsse_common/idsse/common/schema/new_data.json @@ -63,14 +63,18 @@ "field": { "type": "object", "patternProperties": { - "^(\\d{4})-0?(\\d+)-0?(\\d+)[T ]0?(\\d+):0?(\\d+):0?(\\d+)$": {"$ref": "field.json#/FieldList"}, + "^(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}(?:\\.\\d*)?)((-(\\d{2}):(\\d{2})|Z)?)$": { + "$ref": "field.json#/FieldList" + }, "[^0-9.+-:TZ]": {"not": {}} } }, "missing": { "type": "object", "patternProperties": { - "^(\\d{4})-0?(\\d+)-0?(\\d+)[T ]0?(\\d+):0?(\\d+):0?(\\d+)$": {"$ref": "field.json#/FieldList"}, + "^(\\d{4})-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}(?:\\.\\d*)?)((-(\\d{2}):(\\d{2})|Z)?)$": { + "$ref": "field.json#/FieldList" + }, "[^0-9.+-:TZ]": {"not": {}} } } diff --git a/python/idsse_common/idsse/common/schema/translated_data.json b/python/idsse_common/idsse/common/schema/translated_data.json index 27193549..f4719dab 100644 --- a/python/idsse_common/idsse/common/schema/translated_data.json +++ b/python/idsse_common/idsse/common/schema/translated_data.json @@ -1,4 +1,50 @@ { + "SingleTranslatedData": { + "description": "Specification of the data and thresholds that would result in an impact", + "type": "object", + "properties": { + "name": {"type": "string"}, + "arealPercentage": {"type": "number"}, + "duration": {"type": "number"}, + "product": {"$ref": "product.json#/Product"}, + "field": {"type": "string"}, + "region": {"$ref": "region.json#/Region"}, + "units": {"type": "string"}, + "relational": {"type": "string"}, + "thresh": {"type": "number"}, + "mapping": { + "type": "object", + "properties": { + "min": {"type": "number"}, + "max": {"type": "number"}, + "clip": { + "oneOf" : [ + {"type": "boolean"}, + {"type": "string", "enum": ["true", "false"]} + ] + } + }, + "required": [ + "min", + "max", + "clip" + ] + } + }, + "required": [ + "name", + "arealPercentage", + "duration", + "product", + "field", + "region", + "units", + "relational", + "thresh", + "mapping" + ] + }, + "TranslatedData": { "description": "Specification of the data and thresholds that would result in an impact", "type": "object", diff --git a/python/idsse_common/test/test_validate_schema.py b/python/idsse_common/test/test_validate_schema.py index ba0d4854..06261d40 100644 --- a/python/idsse_common/test/test_validate_schema.py +++ b/python/idsse_common/test/test_validate_schema.py @@ -50,7 +50,19 @@ def new_data_validator() -> Validator: @fixture -def data_message() -> dict: +def das_web_request_validator() -> Validator: + schema_name = 'das_web_request_schema' + return get_validator(schema_name) + + +@fixture +def das_web_response_validator() -> Validator: + schema_name = 'das_web_response_schema' + return get_validator(schema_name) + + +@fixture +def das_data_message() -> dict: return { "sourceType": "join", "sourceObj": { @@ -413,6 +425,114 @@ def new_issue_message() -> dict: } +@fixture +def das_web_request_message() -> dict: + return { + "issueDt": "2023-01-10T08:00:00.000Z", + "dataRequest": "A AND B", + "parts": [ + { + "name": "A", + "duration": 0, + "arealPercentage": 0, + "product": "NBM", + "field": "WINDSPEED", + "units": "MilesPerHour", + "region": "CO", + "relational": "GREATER THAN", + "thresh": 5, + "mapping": { + "min": 0.0, + "max": 20.0, + "clip": "true" + } + }, + { + "name": "B", + "duration": 0, + "arealPercentage": 0, + "product": "NBM", + "field": "TEMPERATURE", + "units": "Fahrenheit", + "region": "CO", + "relational": "LESS THAN OR EQUAL", + "thresh": 30, + "mapping": { + "min": 15.0, + "max": 45.0, + "clip": "true" + } + } + ], + "valids": [ + "2023-01-11T06:00:00.000Z", + "2023-01-11T07:00:00.000Z", + "2023-01-11T08:00:00.000Z", + "2023-01-11T09:00:00.000Z", + "2023-01-11T10:00:00.000Z", + "2023-01-11T11:00:00.000Z", + "2023-01-11T12:00:00.000Z", + "2023-01-11T13:00:00.000Z", + "2023-01-11T14:00:00.000Z", + "2023-01-11T15:00:00.000Z", + "2023-01-11T16:00:00.000Z", + "2023-01-11T17:00:00.000Z", + "2023-01-11T18:00:00.000Z" + ], + "bbox": { + "botLeft": [910, 829], + "topRight": [1010, 929] + } + } + + +@fixture +def das_web_response_message() -> dict: + return { + "issueDt": "2023-01-10T08:00:00.000Z", + "dataRequest": "A", + "parts": [ + { + "name": "A", + "duration": 0, + "arealPercentage": 0, + "product": "NBM", + "field": "WINDSPEED", + "units": "MilesPerHour", + "region": "CO", + "relational": "GREATER THAN", + "thresh": 5, + "mapping": { + "min": 0.0, + "max": 20.0, + "clip": "true" + } + } + ], + "valids": [ + "2023-01-11T06:00:00.000Z", + "2023-01-11T18:00:00.000Z" + ], + "bbox": { + "botLeft": [100, 102], + "topRight": [200, 202] + }, + "data": { + "2023-01-11T06:00:00.000Z": [ + [1, 2, 3], + [4, 5, 6], + [7, 8, 9] + ], + "2023-01-11T18:00:00.000Z": [ + [9, 8, 7], + [6, 5, 4], + [3, 2, 1] + ], + "scale": 10 + } + } + + # tests def test_validate_das_issue_request(available_data_validator: Validator): message = {'sourceType': 'issue', @@ -590,21 +710,21 @@ def test_validate_das_bad_opr_with_single_source_request(data_request_validator: def test_validate_das_opr_with_multi_sources_request(data_request_validator: Validator, - data_message: dict): + das_data_message: dict): # this is an example of a logical join operator, the operator itself is not validated try: - data_request_validator.validate(data_message) + data_request_validator.validate(das_data_message) except ValidationError as exc: assert False, f'Validate message raised an exception {exc}' def test_validate_das_bad_opr_with_multi_sources_request(data_request_validator: Validator, - data_message: dict): + das_data_message: dict): # one of the operator object does not contains 'source', has 'not_source' instead - mapping_opr = data_message['sourceObj']['sources'][1]['sourceObj'] + mapping_opr = das_data_message['sourceObj']['sources'][1]['sourceObj'] mapping_opr['not_source'] = mapping_opr.pop('source') with raises(ValidationError): - data_request_validator.validate(data_message) + data_request_validator.validate(das_data_message) def test_validate_simple_criteria_message(criteria_validator: Validator, @@ -660,6 +780,13 @@ def test_validate_event_port_message(event_port_validator: Validator, assert False, f'Validate message raised an exception {exc}' +def test_validate_event_port_message_without_results(event_port_validator: Validator, + simple_event_port_message: dict): + simple_event_port_message.pop('riskResults') + with raises(ValidationError): + event_port_validator.validate(simple_event_port_message) + + def test_validate_event_port_message_with_bad_geo_dist(event_port_validator: Validator, simple_event_port_message: dict): criteria_geo_dist = simple_event_port_message['riskResults'][0]['data'][0]['geoDist'] @@ -765,3 +892,64 @@ def test_validate_new_issue_data_message_bad_valid_string(new_data_validator: Va new_issue_message['field']['Not a string rep of a valid date'] = sample_fields with raises(ValidationError): new_data_validator.validate(new_issue_message) + + +def test_validate_das_web_request_message(das_web_request_validator: Validator, + das_web_request_message: dict): + try: + das_web_request_validator.validate(das_web_request_message) + except ValidationError as exc: + assert False, f'Validate message raised an exception {exc}' + + +def test_validate_das_web_request_message_bad_bbox(das_web_request_validator: Validator, + das_web_request_message: dict): + # replace the bottom left int coordinate with a float + das_web_request_message['bbox']['botLeft'][0] = 1.2 + with raises(ValidationError): + das_web_request_validator.validate(das_web_request_message) + + +def test_validate_das_web_request_message_multi_product(das_web_request_validator: Validator, + das_web_request_message: dict): + # replace the single product with multiple product structure (like in criteria and eventPort) + das_web_request_message['parts'][0]['product'] = { + "fcst": [ + "NBM" + ] + } + with raises(ValidationError): + das_web_request_validator.validate(das_web_request_message) + + +def test_validate_das_web_response_message(das_web_response_validator: Validator, + das_web_response_message: dict): + try: + das_web_response_validator.validate(das_web_response_message) + except ValidationError as exc: + assert False, f'Validate message raised an exception {exc}' + + +def test_validate_das_web_response_message_without_data(das_web_response_validator: Validator, + das_web_response_message: dict): + # remove data + das_web_response_message.pop('data') + with raises(ValidationError): + das_web_response_validator.validate(das_web_response_message) + + +def test_validate_das_web_response_message_bad_data_key(das_web_response_validator: Validator, + das_web_response_message: dict): + # add to data a key that is not scale, offset, datetime + das_web_response_message['data']['not scale/offset or datetime'] = 3 + with raises(ValidationError): + das_web_response_validator.validate(das_web_response_message) + + +def test_validate_das_web_response_message_bad_data(das_web_response_validator: Validator, + das_web_response_message: dict): + # add to data a key that is not scale, offset, datetime + validDt = das_web_response_message['valids'][0] + das_web_response_message['data'][validDt] = [['strings'], ['not', 'numbers']] + with raises(ValidationError): + das_web_response_validator.validate(das_web_response_message)