diff --git a/.pullapprove.yml b/.pullapprove.yml index f432fada2..f8817af75 100644 --- a/.pullapprove.yml +++ b/.pullapprove.yml @@ -8,13 +8,13 @@ pullapprove_conditions: - condition: "not any(['release' in labels, 'hotfix' in labels])" unmet_status: success explanation: "Releases, security updates and hotfixes are allowed to merge in integration, staging and/or master" -- condition: "base.ref == 'develop' or '[Snyk]' in title" +- condition: "base.ref == 'staging' or '[Snyk]' in title" unmet_status: failure - explanation: "No PRs from unknown branches can be merged into integration, staging or master" + explanation: "No PRs from unknown branches can be merged into master" - condition: "'on hold' not in labels" unmet_status: pending explanation: "PR review is on hold" -- condition: "'*[tT]ravis*' in check_runs.successful" +- condition: "'*[tT]ravis*' in check_runs.success" unmet_status: failure explanation: "Travis CI tests must pass before review starts" @@ -27,15 +27,8 @@ notifications: - when: pullapprove.approved if: "author_association == 'CONTRIBUTOR'" # Only contributing authors get this message comment: "The review is completed. Thanks @{{ author }}!" -- when: pull_request.opened - if: "'content' in labels and 'major' in body" # Notify the DCP for a major update - comment: | - Major metadata schema update, pinging @aaclan-ebi @NoopDog - @hannes-ucsc @danxmoran . If no one objects, this will be merged on the - following 5 working days. groups: - snyk-security-patches: conditions: - "'[Snyk]' in title" @@ -43,15 +36,99 @@ groups: reviewers: users: - ESapenaVentura - - rolando-ebi + - aaclan-ebi + - jacobwindsor + - yusra-haider + reviews: + required: 1 + request_order: shuffle + + azul: + conditions: + - "'*[Tt]ravis*' in check_runs.successful" + - "base.ref == 'master'" + - "'content' in labels" + reviewers: + users: + - hannes-ucsc + reviews: + required: 1 + request: 1 + request_order: shuffle + + browser: + conditions: + - "'*[Tt]ravis*' in check_runs.successful" + - "base.ref == 'master'" + - "'content' in labels" + reviewers: + users: + - NoopDog + reviews: + required: 1 + request: 1 + request_order: shuffle + + pipelines: + conditions: + - "'*[Tt]ravis*' in check_runs.successful" + - "base.ref == 'master'" + - "'content' in labels" + reviewers: + users: + - kbergin reviews: required: 1 + request: 1 + request_order: shuffle + + data_import: + conditions: + - "'*[Tt]ravis*' in check_runs.successful" + - "base.ref == 'master'" + - "'content' in labels" + reviewers: + users: + - aherbst-broad + - JeffKorte + - danicham + reviews: + required: 1 + request: 1 + request_order: shuffle + + tdr: + conditions: + - "'*[Tt]ravis*' in check_runs.successful" + - "base.ref == 'master'" + - "'content' in labels" + reviewers: + users: + - ruchim + reviews: + required: 1 + request: 1 + request_order: shuffle + + ingest-devs: + conditions: + - "'*[Tt]ravis*' in check_runs.successful" + - "base.ref == 'master'" + - "'content' in labels" + reviewers: + users: + - aaclan-ebi + - jacobwindsor + - yusra-haider + reviews: + required: 1 + request: 1 request_order: shuffle metadata-updates-major-minor: # group name conditions: # the group is activated when all of these statements are true - "'*[Tt]ravis*' in check_runs.successful" - - "base.ref == 'develop'" + - "base.ref == 'staging'" - "'major' in body or 'minor' in body" # major or minor change is indicated in body of PR - "'content' in labels" reviewers: # people who can approve for this group @@ -62,11 +139,12 @@ groups: - rays22 - ami-day - ami-day + - Wkt8 + - Wkt8 - mshadbolt - willrockout - - pnejad - ESapenaVentura - - claymfischer + - rachadele reviews: # settings for approval and reviewer selection required: 2 request_order: shuffle # reviewers will be chosen in a random order @@ -85,11 +163,12 @@ groups: - rays22 - ami-day - ami-day + - Wkt8 + - Wkt8 - mshadbolt - willrockout - - pnejad - ESapenaVentura - - claymfischer + - rachadele reviews: required: 1 request_order: shuffle @@ -105,6 +184,7 @@ groups: - ESapenaVentura - rays22 - ami-day + - Wkt8 reviews: required: 1 - request_order: shuffle \ No newline at end of file + request_order: shuffle diff --git a/changelog.md b/changelog.md index e26a4fcc6..5a9dd3301 100644 --- a/changelog.md +++ b/changelog.md @@ -7,8 +7,49 @@ and (starting with v4.0.0) this project adheres to [Semantic Versioning](http:// ## [Unreleased](https://github.com/HumanCellAtlas/metadata-schema/tree/develop) +### [module/protocol/matrix.json - v1.0.0] - 2021-03-24 +### Added +Added matrix module + +### [type/protocol/analysis/analysis_protocol.json - v9.2.0] - 2021-03-24 +### Changed +Changed 'computational_method' field to optional, added matrix module field + +### [type/file/analysis_file.json - v6.3.0] - 2021-03-24 +### Added +Added optional 'matrix_cell_count' field. + +### [core/file/file_core.json - v6.2.0] - 2021-03-24 +### Added +Added optional file_source field. + +### [type/file/sequence_file.json - v9.3.0] - 2021-03-24 +### Added +Added optional file_source field. + +### [type/file/supplementary_file.json - v2.3.0] - 2021-03-24 +### Added +Added optional file_source field. + +### [type/file/reference_file.json - v3.3.0] - 2021-03-24 +### Added +Added optional file_source field. + +### [type/file/image_file.json - v2.3.0] - 2021-03-24 +### Added +Added optional file_source field. + + ## [Released](https://github.com/HumanCellAtlas/metadata-schema/) +### [type/project/project.json - v14.2.0] - 2021-03-16 +### Added +Added ega_accessions and dbgap_accessions. Fixes #1336. + +### [system/links.json - v3.0.0] - 2021-03-04 +### Changed +Changed protocol_type from string to enum of strings. Fixes #1333 + ### [system/file_descriptor.json - v2.0.0] - 2020-06-29 ### Removed Removed schema major minor versions. diff --git a/docs/jsonBrowser/core.md b/docs/jsonBrowser/core.md index 8e79a42f1..f273d133b 100644 --- a/docs/jsonBrowser/core.md +++ b/docs/jsonBrowser/core.md @@ -18,6 +18,7 @@ file_name | The name of the file. | string | yes | | File name | | R1.fastq.gz format | The format of the file. | string | yes | | File format | | fastq.gz; tif content_description | General description of the contents of the file. | array | no | [See module file_content_ontology](module.md/#file_content_ontology) | Content description | | checksum | MD5 checksum of the file. | string | no | | Checksum | | e09a986c2e630130b1849d4bf9a94c06 +file_source | The source of the file. This is typically an organisation, repository, person or dedicated process. | string | no | | File source | DCP/2 Analysis, Contributor, ArrayExpress, HCA Release, GEO, SCEA, SCP, DCP/1 Matrix Service, LungMAP, Zenodo, Publication | GEO; Contributor ## Protocol core _Information about an intended protocol that was followed in the project._ diff --git a/docs/jsonBrowser/module.md b/docs/jsonBrowser/module.md index 59f573be4..47f44b924 100644 --- a/docs/jsonBrowser/module.md +++ b/docs/jsonBrowser/module.md @@ -39,6 +39,16 @@ assay_type | Type of assay used to detect target. | object | yes | [See module fluorophore | Fluorophore used to detect target. | array | no | | Fluorophore | | Cy5; Alexa 488 channel_label | Channel label used to assay signal. | array | no | | Channel | | 1; A +## Matrix +_Information relating to generation of matrices_ + +Location: module/protocol/matrix.json + +Property name | Description | Type | Required? | Object reference? | User friendly name | Allowed values | Example +--- | --- | --- | --- | --- | --- | --- | --- +data_normalization_methods | Method(s) used to normalize data in the matrix | array | no | | Data normalization method(s) | CPM (counts per million), TPM (transcripts per kilobase million), RPKM (reads per kilobase of exon per million reads mapped), FPKM (fragments per kilobase of exon per million fragments mapped), DESeq2’s median of ratios, TMM (EdgeR’s trimmed mean of M values), SF (size factor), UQ (Upper quartile), Downsampling, other, unknown | +derivation_process | Type of computational tool used in generating the matrix object. | array | no | | Derivation process | alignment, quantification, peak calling, peak annotation, gene filtering, cell filtering, merging, cell calling, ambient RNA correction, doublet removal, batch correction, depth normalization, other | + ## File content ontology _A term that describes the contents of a file._ diff --git a/docs/jsonBrowser/required_fields.md b/docs/jsonBrowser/required_fields.md index 9de6889d5..8bf069fe0 100644 --- a/docs/jsonBrowser/required_fields.md +++ b/docs/jsonBrowser/required_fields.md @@ -86,7 +86,6 @@ Property name | Description | Type | Object reference? | User friendly name | Al schema_type | The type of the metadata schema entity. | string | | | protocol | protocol_core | Core protocol-level information. | object | [See core protocol_core](core.md/#protocol_core) | Protocol core | | type | The type of protocol. | object | [See module process_type_ontology](module.md/#process_type_ontology) | Protocol type | | -computational_method | A URI to a versioned workflow and versioned execution environment in a GA4GH-compliant repository. | string | | Computational method | | SmartSeq2SingleCell; 10x ### Aggregate generation protocol Property name | Description | Type | Object reference? | User friendly name | Allowed values | Example --- | --- | --- | --- | --- | --- | --- @@ -216,6 +215,8 @@ Property name | Description | Type | Object reference? | User friendly name | Al probe_label | The label of a probe used to detect target in this experiment. | string | | Probe label | | ACTA1; cFos target_label | An identifier for the target molecule. | string | | Target label | | CHEBI:85345; ENSG00000170345 assay_type | Type of assay used to detect target. | object | [See module process_type_ontology](module.md/#process_type_ontology) | Assay type | | MERFISH; in situ sequencing +### Matrix +_There are no required properties in schema Matrix_ ### File content ontology Property name | Description | Type | Object reference? | User friendly name | Allowed values | Example --- | --- | --- | --- | --- | --- | --- diff --git a/docs/jsonBrowser/type.md b/docs/jsonBrowser/type.md index c97669a89..844d0713f 100644 --- a/docs/jsonBrowser/type.md +++ b/docs/jsonBrowser/type.md @@ -56,6 +56,7 @@ Property name | Description | Type | Required? | Object reference? | User friend schema_type | The type of the metadata schema entity. | string | yes | | | file | provenance | Provenance information provided by the system. | object | no | [See provenance](.md/#provenance) | | | file_core | Core file-level information. | object | yes | [See core file_core](core.md/#file_core) | File core | | +matrix_cell_count | Number of cells analyzed in a matrix file. | integer | no | | Matrix cell count | | 1; 2100 ## Reference file _A reference file used by a secondary reference pipeline._ @@ -139,7 +140,8 @@ schema_type | The type of the metadata schema entity. | string | yes | | | pro provenance | Provenance information provided by the system. | object | no | [See provenance](.md/#provenance) | | | protocol_core | Core protocol-level information. | object | yes | [See core protocol_core](core.md/#protocol_core) | Protocol core | | type | The type of protocol. | object | yes | [See module process_type_ontology](module.md/#process_type_ontology) | Protocol type | | -computational_method | A URI to a versioned workflow and versioned execution environment in a GA4GH-compliant repository. | string | yes | | Computational method | | SmartSeq2SingleCell; 10x +computational_method | A URI to a versioned workflow and versioned execution environment in a GA4GH-compliant repository. | string | no | | Computational method | | SmartSeq2SingleCell; 10x +matrix | Information related to protocols that output a matrix. | object | no | [See module matrix](module.md/#matrix) | Matrix | | ## Aggregate generation protocol _Information about how cultured cells are developed into cell aggregates._ @@ -293,6 +295,8 @@ contributors | People contributing to any aspect of the project. | array | no | supplementary_links | External link(s) pointing to code, supplementary data files, or analysis files associated with the project which will not be uploaded. | array | no | | Supplementary link(s) | | https://github.com/czbiohub/tabula-muris; http://celltag.org/ publications | Publications resulting from this project. | array | no | [See module publication](module.md/#publication) | Publications | | insdc_project_accessions | An International Nucleotide Sequence Database Collaboration (INSDC) project accession. | array | no | | INSDC project accession | | SRP000000 +ega_accessions | A list of accessions referring to EGA (European Genome-Phenome Archive) datasets or studies. | array | no | | EGA Study/Dataset Accession(s) | | EGAS00000000001; EGAD00000000002 +dbgap_accessions | A list of database of Genotypes and Phenotypes (dbGaP) study accessions. | array | no | | dbGaP Study Accession(s) | | phs001997.v1.p1; phs001836 geo_series_accessions | A Gene Expression Omnibus (GEO) series accession. | array | no | | GEO series accession | | GSE00000 array_express_accessions | An ArrayExpress accession. | array | no | | ArrayExpress accession | | E-AAAA-00 insdc_study_accessions | An International Nucleotide Sequence Database Collaboration (INSDC) study accession. | array | no | | INSDC study accession | | PRJNA000000 diff --git a/infrastructure_testing_files/README.md b/infrastructure_testing_files/README.md new file mode 100644 index 000000000..3b42d5275 --- /dev/null +++ b/infrastructure_testing_files/README.md @@ -0,0 +1,9 @@ +## Test data + +The spreadsheet at the root of this folder represents a full hca project, with all possible entities included (Except imaging: WIP) + +This spreadsheet will be used to test metadata moving through ingest and will be updated on every release to the integration environment (Every thursday) + +The actual data is mock data and is stored at the hca-util upload area named `test-data-area` + +Instructions on generating the mock data coming soon \ No newline at end of file diff --git a/infrastructure_testing_files/test_metadata_hca_spreadsheet.xlsx b/infrastructure_testing_files/test_metadata_hca_spreadsheet.xlsx new file mode 100644 index 000000000..3d56cdcfd Binary files /dev/null and b/infrastructure_testing_files/test_metadata_hca_spreadsheet.xlsx differ diff --git a/infrastructure_testing_files/test_metadata_hca_spreadsheet_imaging.xlsx b/infrastructure_testing_files/test_metadata_hca_spreadsheet_imaging.xlsx new file mode 100644 index 000000000..e34dba334 Binary files /dev/null and b/infrastructure_testing_files/test_metadata_hca_spreadsheet_imaging.xlsx differ diff --git a/infrastructure_testing_files/test_metadata_hca_spreadsheet_no_analysis.xlsx b/infrastructure_testing_files/test_metadata_hca_spreadsheet_no_analysis.xlsx new file mode 100644 index 000000000..7afff1fb5 Binary files /dev/null and b/infrastructure_testing_files/test_metadata_hca_spreadsheet_no_analysis.xlsx differ diff --git a/json_schema/core/file/file_core.json b/json_schema/core/file/file_core.json index de4b837e1..995a291da 100644 --- a/json_schema/core/file/file_core.json +++ b/json_schema/core/file/file_core.json @@ -49,6 +49,26 @@ "type": "string", "user_friendly": "Checksum", "example": "e09a986c2e630130b1849d4bf9a94c06" - } + }, + "file_source": { + "description": "The source of the file. This is typically an organisation, repository, person or dedicated process.", + "type": "string", + "enum":[ + "DCP/2 Analysis", + "Contributor", + "ArrayExpress", + "HCA Release", + "GEO", + "SCEA", + "SCP", + "DCP/1 Matrix Service", + "LungMAP", + "Zenodo", + "Publication" + ], + "user_friendly": "File source", + "guidelines": "Should be one of: DCP/2 Analysis, Contributor, ArrayExpress, HCA Release, GEO, SCEA, SCP, DCP/1 Matrix Service, LungMAP, Zenodo, Publication", + "example": "GEO; Contributor" + } } } diff --git a/json_schema/module/protocol/matrix.json b/json_schema/module/protocol/matrix.json new file mode 100644 index 000000000..e2ff9bdd0 --- /dev/null +++ b/json_schema/module/protocol/matrix.json @@ -0,0 +1,67 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Information relating to generation of matrices", + "additionalProperties": false, + "title": "Matrix", + "name": "matrix", + "type": "object", + "properties": { + "describedBy": { + "description": "The URL reference to the schema.", + "type": "string", + "pattern" : "^(http|https)://schema.(.*?)humancellatlas.org/module/protocol/(([0-9]{1,}.[0-9]{1,}.[0-9]{1,})|([a-zA-Z]*?))/matrix" + }, + "schema_version": { + "description": "The version number of the schema in major.minor.patch format.", + "type": "string", + "pattern": "^[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$", + "example": "4.6.1" + }, + "data_normalization_methods": { + "description": "Method(s) used to normalize data in the matrix", + "type": "array", + "user_friendly": "Data normalization method(s)", + "items": { + "type": "string", + "enum": [ + "CPM (counts per million)", + "TPM (transcripts per kilobase million)", + "RPKM (reads per kilobase of exon per million reads mapped)", + "FPKM (fragments per kilobase of exon per million fragments mapped)", + "DESeq2’s median of ratios", + "TMM (EdgeR’s trimmed mean of M values)", + "SF (size factor)", + "UQ (Upper quartile)", + "Downsampling", + "other", + "unknown" + ] + }, + "guidelines": "Should be one of: CPM (counts per million), TPM (transcripts per kilobase million), RPKM(reads per kilobase of exon per million reads mapped), FPKM (fragments per kilobase of exon per million fragments mapped), DESeq2’s median of ratios, TMM (EdgeR’s trimmed mean of M values), SF (size factor), UQ (Upper quartile), Downsampling, unknown, other." + }, + "derivation_process": { + "description": "Type of computational tool used in generating the matrix object.", + "type": "array", + "user_friendly": "Derivation process", + "guidelines": "Should be one of: alignment, quantification, peak calling, peak annotation, gene filtering, cell filtering, merging, cell calling, ambient RNA correction, doublet removal, batch correction, depth normalization, other.", + "items": { + "type": "string", + "enum": [ + "alignment", + "quantification", + "peak calling", + "peak annotation", + "gene filtering", + "cell filtering", + "merging", + "cell calling", + "ambient RNA correction", + "doublet removal", + "batch correction", + "depth normalization", + "other" + ] + } + } + } +} diff --git a/json_schema/system/file_descriptor.json b/json_schema/system/file_descriptor.json index d823eba6f..6d62f3d73 100644 --- a/json_schema/system/file_descriptor.json +++ b/json_schema/system/file_descriptor.json @@ -20,7 +20,7 @@ "describedBy": { "description": "The URL reference to the schema.", "type": "string", - "pattern" : "^https?://schema(\\.[a-z]+)?\\.humancellatlas\\.org/system/(([0-9]{1,}\\.[0-9]{1,}\\.[0-9]{1,})|([a-zA-Z]*?))/file_descriptor$" + "pattern" : "^https?://schema.(.*)?humancellatlas.org/system/(([0-9]{1,}.[0-9]{1,}.[0-9]{1,})|([a-zA-Z]*?))/file_descriptor$" }, "schema_version": { "description": "The version number of the schema in major.minor.patch format.", diff --git a/json_schema/system/links.json b/json_schema/system/links.json index 085635f69..e537c246b 100644 --- a/json_schema/system/links.json +++ b/json_schema/system/links.json @@ -72,7 +72,20 @@ "description": "The concrete type of protocol entity.", "example": "enrichment_protocol", "user_friendly": "Protocol type", - "type": "string" + "type": "string", + "enum": [ + "analysis_protocol", + "aggregate_generation_protocol", + "collection_protocol", + "differentiation_protocol", + "dissociation_protocol", + "enrichment_protocol", + "ipsc_induction_protocol", + "imaging_preparation_protocol", + "imaging_protocol", + "library_preparation_protocol", + "sequencing_protocol" + ] }, "protocol_id": { "description": "UUID of the protocol entity.", diff --git a/json_schema/type/file/analysis_file.json b/json_schema/type/file/analysis_file.json index a97d161c4..5cdfc939f 100644 --- a/json_schema/type/file/analysis_file.json +++ b/json_schema/type/file/analysis_file.json @@ -39,6 +39,15 @@ "type": "object", "$ref": "core/file/file_core.json", "user_friendly": "File core" + }, + "matrix_cell_count": { + "description": "Number of cells analyzed in a matrix file.", + "type": "integer", + "maximum": 1000000000, + "minimum": 0, + "example": "1; 2100", + "user_friendly": "Matrix cell count", + "guidelines": "If the analysis file is a matrix containing cells, enter the exact number of cells in the matrix." } } } diff --git a/json_schema/type/project/project.json b/json_schema/type/project/project.json index 53b7d6c6f..232f06b29 100644 --- a/json_schema/type/project/project.json +++ b/json_schema/type/project/project.json @@ -77,6 +77,28 @@ "user_friendly": "INSDC project accession", "guidelines": "Enter accession if project has been archived. Accession can be from the DDBJ, NCBI, or EMBL-EBI and must start with DRP, SRP, or ERP, respectively." }, + "ega_accessions": { + "description": "A list of accessions referring to EGA (European Genome-Phenome Archive) datasets or studies.", + "type": "array", + "items": { + "type": "string", + "pattern": "EGA[DS][0-9]{11}" + }, + "example": "EGAS00000000001; EGAD00000000002", + "guidelines": "Enter any EGA study or dataset accessions that relate to the project. Should start with EGAD or EGAS, study accession preferred.", + "user_friendly": "EGA Study/Dataset Accession(s)" + }, + "dbgap_accessions": { + "description": "A list of database of Genotypes and Phenotypes (dbGaP) study accessions.", + "type": "array", + "items": { + "type": "string", + "pattern": "phs[0-9]{6}(\\.v[0-9])?(\\.p[0-9])?" + }, + "example": "phs001997.v1.p1; phs001836", + "guidelines": "Enter any dbGaP study accessions that relate to this project. Should start with phs, can contain the specific version information.", + "user_friendly": "dbGaP Study Accession(s)" + }, "geo_series_accessions": { "description": "A Gene Expression Omnibus (GEO) series accession.", "type": "array", @@ -130,4 +152,4 @@ "user_friendly": "Funding source(s)" } } -} \ No newline at end of file +} diff --git a/json_schema/type/protocol/analysis/analysis_protocol.json b/json_schema/type/protocol/analysis/analysis_protocol.json index 7e4a2842c..4ddcbfa40 100644 --- a/json_schema/type/protocol/analysis/analysis_protocol.json +++ b/json_schema/type/protocol/analysis/analysis_protocol.json @@ -6,7 +6,6 @@ "describedBy", "schema_type", "protocol_core", - "computational_method", "type" ], "title": "Analysis protocol", @@ -53,6 +52,12 @@ "type": "string", "user_friendly": "Computational method", "example": "SmartSeq2SingleCell; 10x" + }, + "matrix": { + "description": "Information related to protocols that output a matrix.", + "type": "object", + "$ref": "module/protocol/matrix.json", + "user_friendly": "Matrix" } } } diff --git a/json_schema/versions.json b/json_schema/versions.json index d1dc21c83..0f6c1aaab 100644 --- a/json_schema/versions.json +++ b/json_schema/versions.json @@ -1,12 +1,12 @@ { - "last_update_date": "2020-06-29T16:20:46Z", + "last_update_date": "2021-03-25T14:17:02Z", "version_numbers": { "core": { "biomaterial": { "biomaterial_core": "8.2.0" }, "file": { - "file_core": "6.1.1" + "file_core": "6.2.0" }, "process": { "process_core": "10.0.0" @@ -74,13 +74,14 @@ }, "protocol": { "channel": "2.0.4", + "matrix": "1.0.0", "probe": "1.1.1" } }, "system": { "file_descriptor": "2.0.0", "license": "1.0.0", - "links": "2.1.1", + "links": "3.0.0", "provenance": "1.1.0" }, "type": { @@ -93,11 +94,11 @@ "specimen_from_organism": "10.4.0" }, "file": { - "analysis_file": "6.2.0", - "image_file": "2.2.0", - "reference_file": "3.2.0", - "sequence_file": "9.2.0", - "supplementary_file": "2.2.0" + "analysis_file": "6.3.0", + "image_file": "2.3.0", + "reference_file": "3.3.0", + "sequence_file": "9.3.0", + "supplementary_file": "2.3.0" }, "process": { "analysis": { @@ -106,11 +107,11 @@ "process": "9.2.0" }, "project": { - "project": "14.1.0" + "project": "14.2.0" }, "protocol": { "analysis": { - "analysis_protocol": "9.1.0" + "analysis_protocol": "9.2.0" }, "biomaterial_collection": { "aggregate_generation_protocol": "2.1.0", diff --git a/src/human_readable_json.py b/src/human_readable_json.py index 0d1aa170e..abe67797a 100644 --- a/src/human_readable_json.py +++ b/src/human_readable_json.py @@ -117,13 +117,19 @@ def generateMarkdown(self, schemas, entity_type): # if link is not "": # print(schema["title"] + "\t "+ property + "\t"+ link) + if "enum" in schema["properties"][property]: + enum_values = ", ".join(schema["properties"][property]["enum"]) + elif "enum" in schema["properties"][property].get("items", ""): + enum_values = ", ".join(schema["properties"][property]["items"]["enum"]) + else: + enum_values = "" file.write(property + " | "+ (schema["properties"][property]["description"] if "description" in schema["properties"][property] else "") + " | " + (schema["properties"][property]["type"] if "type" in schema["properties"][property] else "") + " | " + ("yes" if property in required else "no") + " | " + link + " | " + (schema["properties"][property]["user_friendly"] if "user_friendly" in schema["properties"][property] else "") + " | " + - (", ".join(schema["properties"][property]["enum"]) if "enum" in schema["properties"][property] else "") + " | " + + enum_values + " | " + (str(schema["properties"][property]["example"]) if "example" in schema["properties"][property] else "") + "\n") if property in required: diff --git a/src/release_prepare.py b/src/release_prepare.py index a6218aa8f..1df17d36f 100644 --- a/src/release_prepare.py +++ b/src/release_prepare.py @@ -53,7 +53,7 @@ def updateVersions(self, log_content): if not log_content[val][self.version_column]: schema = log_content[val][self.schema_column] change_type = log_content[val][self.type_column] - version_options = options(self.schemafolder, schema,change_type,independent_schemas) + version_options = options(self.schemafolder, schema, change_type, independent_schemas) versionUpdater = VersionUpdater(version_options) versionUpdates = versionUpdater.updateVersions(True) diff --git a/src/schema_linter.py b/src/schema_linter.py index 418894558..13481a089 100644 --- a/src/schema_linter.py +++ b/src/schema_linter.py @@ -333,7 +333,7 @@ def get_json_from_file(self, filename, warn = False): # define the proper OLS API URL. arguments = argument_parser().parse_args(sys.argv[1:]) environment = OLS_ENVIRONMENT[arguments.environment] - ols_api = 'https://ontology.{}.data.humancellatlas.org/api'.format(environment) + ols_api = 'https://ontology.{}.archive.data.humancellatlas.org/api'.format(environment) schema_path = '../json_schema' if cwd == 'src' else 'json_schema' jsons = [os.path.join(dirpath, f) diff --git a/tests/package-lock.json b/tests/package-lock.json index 8df4c8e4e..eb8733261 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -459,9 +459,9 @@ } }, "bl": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", - "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -1123,7 +1123,7 @@ }, "event-stream": { "version": "3.3.5", - "resolved": "http://registry.npmjs.org/event-stream/-/event-stream-3.3.5.tgz", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.5.tgz", "integrity": "sha512-vyibDcu5JL20Me1fP734QBH/kenBGLZap2n0+XXM7mvuUPzJ20Ydqj1aKcIeMdri1p+PU+4yAKugjN8KCVst+g==", "requires": { "duplexer": "^0.1.1", @@ -1613,9 +1613,9 @@ "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "invert-kv": { "version": "1.0.0", @@ -1882,9 +1882,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.assign": { "version": "4.2.0", @@ -1969,7 +1969,7 @@ }, "map-stream": { "version": "0.0.7", - "resolved": "http://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg=" }, "md5": { @@ -3678,7 +3678,7 @@ }, "split": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/split/-/split-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", "requires": { "through": "2" @@ -3722,7 +3722,7 @@ }, "stream-combiner": { "version": "0.2.2", - "resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", "requires": { "duplexer": "~0.1.1", diff --git a/tests/schema.test.js b/tests/schema.test.js index 5a58b36ee..d56394c71 100644 --- a/tests/schema.test.js +++ b/tests/schema.test.js @@ -18,7 +18,7 @@ let baseSchemaPath = path.join(__dirname, '..', schemaDirs); // set the example data path let baseDataPath = path.join(__dirname, '..', exampleDataDirs); -let ajvOptions = {baseSchemaPath: baseSchemaPath, extendRefs: true, olsApiBaseUrl: 'http://ontology.dev.data.humancellatlas.org/api'}; +let ajvOptions = {baseSchemaPath: baseSchemaPath, extendRefs: true, olsApiBaseUrl: 'http://ontology.archive.data.humancellatlas.org/api'}; let validator = new ElixirValidator([GraphRestriction], ajvOptions); var ajv = new Ajv(ajvOptions); @@ -36,7 +36,8 @@ var tests = [ {args: ['type/biomaterial/specimen_from_organism.json', 'biomaterial/test_fail_specimen_0.json'], expectedErrors: 1}, {args: ['type/process/analysis/analysis_process.json', 'process/test_pass_analysis_process.json'], expectedErrors: 0}, {args: ['type/process/analysis/analysis_process.json', 'process/test_pass_new_analysis_process.json'], expectedErrors: 0}, - {args: ['system/links.json', 'system/test_pass_links.json'], expectedErrors: 0} + {args: ['system/links.json', 'system/test_pass_links.json'], expectedErrors: 0}, + {args: ['system/links.json', 'system/test_fail_links.json'], expectedErrors: 10} ]; @@ -101,7 +102,7 @@ describe ('Testing example data validates against schemas', function () { } }); - }) + }).timeout(10000) }); diff --git a/tests/schema_test_files/project/test_pass_project_0.json b/tests/schema_test_files/project/test_pass_project_0.json index bc99db1fd..042ac794b 100644 --- a/tests/schema_test_files/project/test_pass_project_0.json +++ b/tests/schema_test_files/project/test_pass_project_0.json @@ -1,6 +1,6 @@ { - "describedBy": "https://schema.dev.data.humancellatlas.org/type/project/5.1.0/project", - "schema_version": "5.1.0", + "describedBy": "https://schema.dev.data.humancellatlas.org/type/project/14.1.0/project", + "schema_version": "14.1.0", "schema_type": "project", "project_core": { "project_short_name": "P1", diff --git a/tests/schema_test_files/system/test_fail_links.json b/tests/schema_test_files/system/test_fail_links.json new file mode 100644 index 000000000..56e781e6f --- /dev/null +++ b/tests/schema_test_files/system/test_fail_links.json @@ -0,0 +1,47 @@ +{ + "describedBy": "https://schema.dev.data.humancellatlas.org/system/9.9.9/links.json", + "schema_type": "links", + "links": [ + { + "link_type": "process_link", + "process_id": "b7a172d6-dbb1-41f3-8ae4-7807e1eca803", + "process_type": "analysis_process", + "inputs": [ + { + "input_type": "sequence_file", + "input_id": "6f725a94-5c81-45e5-8d16-96520aa99703" + } + ], + "outputs": [ + { + "output_type": "analysis_file", + "output_id": "2e072336-b906-4c9b-a475-ca03f51c8452" + } + ], + "protocols": [ + { + "protocol_type": "nonexistent_protocol", + "protocol_id": "4d6f7580-ce81-4a81-9c2c-872fcb23b7cd" + } + ] + }, + { + "link_type": "supplementary_file_link", + "entity": { + "entity_type": "project", + "entity_id": "589b7580-ce81-4a81-9c2c-872fcb23b7cd" + }, + "files": [ + { + "file_type": "supplementary_file", + "file_id": "8abc7580-ce81-4a81-9c2c-872fcb23b7cd" + }, + { + "file_type": "supplementary_file", + "file_id": "cb377580-ce81-4a81-9c2c-872fcb23b7cd" + } + ] + } + ] + +}