Merge pull request #1371 from HumanCellAtlas/staging

Release from staging to master - 2021-03-17

.pullapprove.yml

@@ -8,13 +8,13 @@ pullapprove_conditions:
 - condition: "not any(['release' in labels, 'hotfix' in labels])"
   unmet_status: success
   explanation: "Releases, security updates and hotfixes are allowed to merge in integration, staging and/or master"
-- condition: "base.ref == 'develop' or '[Snyk]' in title"
+- condition: "base.ref == 'staging' or '[Snyk]' in title"
   unmet_status: failure
-  explanation: "No PRs from unknown branches can be merged into integration, staging or master"
+  explanation: "No PRs from unknown branches can be merged into master"
 - condition: "'on hold' not in labels"
   unmet_status: pending
   explanation: "PR review is on hold"
-- condition: "'*[tT]ravis*' in check_runs.successful"
+- condition: "'*[tT]ravis*' in check_runs.success"
   unmet_status: failure
   explanation: "Travis CI tests must pass before review starts"
 
@@ -27,31 +27,108 @@ notifications:
 - when: pullapprove.approved
   if: "author_association == 'CONTRIBUTOR'" # Only contributing authors get this message
   comment: "The review is completed. Thanks @{{ author }}!"
-- when: pull_request.opened
-  if: "'content' in labels and 'major' in body" # Notify the DCP for a major update
-  comment: |
-    Major metadata schema update, pinging @aaclan-ebi @NoopDog
-    @hannes-ucsc @danxmoran . If no one objects, this will be merged on the
-    following 5 working days.
 
 groups:
-
   snyk-security-patches:
     conditions:
     - "'[Snyk]' in title"
     - "'Security' not in labels"
     reviewers:
       users:
       - ESapenaVentura
-      - rolando-ebi
+      - aaclan-ebi
+      - jacobwindsor
+      - yusra-haider
+    reviews:
+      required: 1
+      request_order: shuffle
+
+  azul:
+    conditions:
+    - "'*[Tt]ravis*' in check_runs.successful"
+    - "base.ref == 'master'"
+    - "'content' in labels"
+    reviewers:
+      users:
+      - hannes-ucsc
+    reviews:
+      required: 1
+      request: 1
+      request_order: shuffle
+
+  browser:
+    conditions:
+      - "'*[Tt]ravis*' in check_runs.successful"
+      - "base.ref == 'master'"
+      - "'content' in labels"
+    reviewers:
+      users:
+        - NoopDog
+    reviews:
+      required: 1
+      request: 1
+      request_order: shuffle
+
+  pipelines:
+    conditions:
+    - "'*[Tt]ravis*' in check_runs.successful"
+    - "base.ref == 'master'"
+    - "'content' in labels"
+    reviewers:
+      users:
+      - kbergin
     reviews:
       required: 1
+      request: 1
+      request_order: shuffle
+
+  data_import:
+    conditions:
+    - "'*[Tt]ravis*' in check_runs.successful"
+    - "base.ref == 'master'"
+    - "'content' in labels"
+    reviewers:
+      users:
+      - aherbst-broad
+      - JeffKorte
+      - danicham
+    reviews:
+      required: 1
+      request: 1
+      request_order: shuffle
+
+  tdr:
+    conditions:
+    - "'*[Tt]ravis*' in check_runs.successful"
+    - "base.ref == 'master'"
+    - "'content' in labels"
+    reviewers:
+      users:
+      - ruchim
+    reviews:
+      required: 1
+      request: 1
+      request_order: shuffle
+
+  ingest-devs:
+    conditions:
+    - "'*[Tt]ravis*' in check_runs.successful"
+    - "base.ref == 'master'"
+    - "'content' in labels"
+    reviewers:
+      users:
+      - aaclan-ebi
+      - jacobwindsor
+      - yusra-haider
+    reviews:
+      required: 1
+      request: 1
       request_order: shuffle
 
   metadata-updates-major-minor:  # group name
     conditions:  # the group is activated when all of these statements are true
     - "'*[Tt]ravis*' in check_runs.successful"
-    - "base.ref == 'develop'"
+    - "base.ref == 'staging'"
     - "'major' in body or 'minor' in body" # major or minor change is indicated in body of PR
     - "'content' in labels"
     reviewers:  # people who can approve for this group
@@ -62,11 +139,12 @@ groups:
       - rays22
       - ami-day
       - ami-day
+      - Wkt8
+      - Wkt8
       - mshadbolt
       - willrockout
-      - pnejad
       - ESapenaVentura
-      - claymfischer
+      - rachadele
     reviews:  # settings for approval and reviewer selection
       required: 2
       request_order: shuffle  # reviewers will be chosen in a random order
@@ -85,11 +163,12 @@ groups:
         - rays22
         - ami-day
         - ami-day
+        - Wkt8
+        - Wkt8
         - mshadbolt
         - willrockout
-        - pnejad
         - ESapenaVentura
-        - claymfischer
+        - rachadele
     reviews:
       required: 1
       request_order: shuffle
@@ -105,6 +184,7 @@ groups:
       - ESapenaVentura
       - rays22
       - ami-day
+      - Wkt8
     reviews:
       required: 1
-      request_order: shuffle
+      request_order: shuffle

changelog.md

@@ -7,8 +7,49 @@ and (starting with v4.0.0) this project adheres to [Semantic Versioning](http://
 
 ## [Unreleased](https://github.com/HumanCellAtlas/metadata-schema/tree/develop)
 
+### [module/protocol/matrix.json - v1.0.0] - 2021-03-24
+### Added
+Added matrix module
+
+### [type/protocol/analysis/analysis_protocol.json - v9.2.0] - 2021-03-24
+### Changed
+Changed 'computational_method' field to optional, added matrix module field
+
+### [type/file/analysis_file.json - v6.3.0] - 2021-03-24
+### Added
+Added optional 'matrix_cell_count' field.
+
+### [core/file/file_core.json - v6.2.0] - 2021-03-24
+### Added
+Added optional file_source field.
+
+### [type/file/sequence_file.json - v9.3.0] - 2021-03-24
+### Added
+Added optional file_source field.
+
+### [type/file/supplementary_file.json - v2.3.0] - 2021-03-24
+### Added
+Added optional file_source field.
+
+### [type/file/reference_file.json - v3.3.0] - 2021-03-24
+### Added
+Added optional file_source field.
+
+### [type/file/image_file.json - v2.3.0] - 2021-03-24
+### Added
+Added optional file_source field.
+
+
 ## [Released](https://github.com/HumanCellAtlas/metadata-schema/)
 
+### [type/project/project.json - v14.2.0] - 2021-03-16
+### Added
+Added ega_accessions and dbgap_accessions. Fixes #1336.
+
+### [system/links.json - v3.0.0] - 2021-03-04
+### Changed
+Changed protocol_type from string to enum of strings. Fixes #1333
+
 ### [system/file_descriptor.json - v2.0.0] - 2020-06-29
 ### Removed
 Removed schema major minor versions.

docs/jsonBrowser/core.md

@@ -18,6 +18,7 @@ file_name | The name of the file. | string | yes |  | File name |  | R1.fastq.gz
 format | The format of the file. | string | yes |  | File format |  | fastq.gz; tif
 content_description | General description of the contents of the file. | array | no | [See module  file_content_ontology](module.md/#file_content_ontology) | Content description |  | 
 checksum | MD5 checksum of the file. | string | no |  | Checksum |  | e09a986c2e630130b1849d4bf9a94c06
+file_source | The source of the file. This is typically an organisation, repository, person or dedicated process. | string | no |  | File source | DCP/2 Analysis, Contributor, ArrayExpress, HCA Release, GEO, SCEA, SCP, DCP/1 Matrix Service, LungMAP, Zenodo, Publication | GEO; Contributor
 
 ## Protocol core<a name='Protocol core'></a>
 _Information about an intended protocol that was followed in the project._

docs/jsonBrowser/module.md

@@ -39,6 +39,16 @@ assay_type | Type of assay used to detect target. | object | yes | [See module
 fluorophore | Fluorophore used to detect target. | array | no |  | Fluorophore |  | Cy5; Alexa 488
 channel_label | Channel label used to assay signal. | array | no |  | Channel |  | 1; A
 
+## Matrix<a name='Matrix'></a>
+_Information relating to generation of matrices_
+
+Location: module/protocol/matrix.json
+
+Property name | Description | Type | Required? | Object reference? | User friendly name | Allowed values | Example 
+--- | --- | --- | --- | --- | --- | --- | --- 
+data_normalization_methods | Method(s) used to normalize data in the matrix | array | no |  | Data normalization method(s) | CPM (counts per million), TPM (transcripts per kilobase million), RPKM (reads per kilobase of exon per million reads mapped), FPKM (fragments per kilobase of exon per million fragments mapped), DESeq2’s median of ratios, TMM (EdgeR’s trimmed mean of M values), SF (size factor), UQ (Upper quartile), Downsampling, other, unknown | 
+derivation_process | Type of computational tool used in generating the matrix object. | array | no |  | Derivation process | alignment, quantification, peak calling, peak annotation, gene filtering, cell filtering, merging, cell calling, ambient RNA correction, doublet removal, batch correction, depth normalization, other | 
+
 ## File content ontology<a name='File content ontology'></a>
 _A term that describes the contents of a file._
 

docs/jsonBrowser/required_fields.md

@@ -86,7 +86,6 @@ Property name | Description | Type | Object reference? | User friendly name | Al
 schema_type | The type of the metadata schema entity. | string |  |  | protocol | 
 protocol_core | Core protocol-level information. | object | [See core  protocol_core](core.md/#protocol_core) | Protocol core |  | 
 type | The type of protocol. | object | [See module  process_type_ontology](module.md/#process_type_ontology) | Protocol type |  | 
-computational_method | A URI to a versioned workflow and versioned execution environment in a GA4GH-compliant repository. | string |  | Computational method |  | SmartSeq2SingleCell; 10x
 ### Aggregate generation protocol
 Property name | Description | Type | Object reference? | User friendly name | Allowed values | Example 
 --- | --- | --- | --- | --- | --- | --- 
@@ -216,6 +215,8 @@ Property name | Description | Type | Object reference? | User friendly name | Al
 probe_label | The label of a probe used to detect target in this experiment. | string |  | Probe label |  | ACTA1; cFos
 target_label | An identifier for the target molecule. | string |  | Target label |  | CHEBI:85345; ENSG00000170345
 assay_type | Type of assay used to detect target. | object | [See module  process_type_ontology](module.md/#process_type_ontology) | Assay type |  | MERFISH; in situ sequencing
+### Matrix<a name='Matrix'></a>
+_There are no required properties in schema Matrix_
 ### File content ontology<a name='File content ontology'></a>
 Property name | Description | Type | Object reference? | User friendly name | Allowed values | Example 
 --- | --- | --- | --- | --- | --- | --- 

docs/jsonBrowser/type.md

@@ -56,6 +56,7 @@ Property name | Description | Type | Required? | Object reference? | User friend
 schema_type | The type of the metadata schema entity. | string | yes |  |  | file | 
 provenance | Provenance information provided by the system. | object | no | [See   provenance](.md/#provenance) |  |  | 
 file_core | Core file-level information. | object | yes | [See core  file_core](core.md/#file_core) | File core |  | 
+matrix_cell_count | Number of cells analyzed in a matrix file. | integer | no |  | Matrix cell count |  | 1; 2100
 
 ## Reference file
 _A reference file used by a secondary reference pipeline._
@@ -139,7 +140,8 @@ schema_type | The type of the metadata schema entity. | string | yes |  |  | pro
 provenance | Provenance information provided by the system. | object | no | [See   provenance](.md/#provenance) |  |  | 
 protocol_core | Core protocol-level information. | object | yes | [See core  protocol_core](core.md/#protocol_core) | Protocol core |  | 
 type | The type of protocol. | object | yes | [See module  process_type_ontology](module.md/#process_type_ontology) | Protocol type |  | 
-computational_method | A URI to a versioned workflow and versioned execution environment in a GA4GH-compliant repository. | string | yes |  | Computational method |  | SmartSeq2SingleCell; 10x
+computational_method | A URI to a versioned workflow and versioned execution environment in a GA4GH-compliant repository. | string | no |  | Computational method |  | SmartSeq2SingleCell; 10x
+matrix | Information related to protocols that output a matrix. | object | no | [See module  matrix](module.md/#matrix) | Matrix |  | 
 
 ## Aggregate generation protocol
 _Information about how cultured cells are developed into cell aggregates._
@@ -293,6 +295,8 @@ contributors | People contributing to any aspect of the project. | array | no |
 supplementary_links | External link(s) pointing to code, supplementary data files, or analysis files associated with the project which will not be uploaded. | array | no |  | Supplementary link(s) |  | https://github.com/czbiohub/tabula-muris; http://celltag.org/
 publications | Publications resulting from this project. | array | no | [See module  publication](module.md/#publication) | Publications |  | 
 insdc_project_accessions | An International Nucleotide Sequence Database Collaboration (INSDC) project accession. | array | no |  | INSDC project accession |  | SRP000000
+ega_accessions | A list of accessions referring to EGA (European Genome-Phenome Archive) datasets or studies. | array | no |  | EGA Study/Dataset Accession(s) |  | EGAS00000000001; EGAD00000000002
+dbgap_accessions | A list of database of Genotypes and Phenotypes (dbGaP) study accessions. | array | no |  | dbGaP Study Accession(s) |  | phs001997.v1.p1; phs001836
 geo_series_accessions | A Gene Expression Omnibus (GEO) series accession. | array | no |  | GEO series accession |  | GSE00000
 array_express_accessions | An ArrayExpress accession. | array | no |  | ArrayExpress accession |  | E-AAAA-00
 insdc_study_accessions | An International Nucleotide Sequence Database Collaboration (INSDC) study accession. | array | no |  | INSDC study accession |  | PRJNA000000

infrastructure_testing_files/README.md

@@ -0,0 +1,9 @@
+## Test data
+
+The spreadsheet at the root of this folder represents a full hca project, with all possible entities included (Except imaging: WIP)
+
+This spreadsheet will be used to test metadata moving through ingest and will be updated on every release to the integration environment (Every thursday)
+
+The actual data is mock data and is stored at the hca-util upload area named `test-data-area`
+
+Instructions on generating the mock data coming soon

json_schema/core/file/file_core.json

@@ -49,6 +49,26 @@
             "type": "string",
             "user_friendly": "Checksum",
             "example": "e09a986c2e630130b1849d4bf9a94c06"
-	    }
+	    },
+        "file_source": {
+            "description": "The source of the file. This is typically an organisation, repository, person or dedicated process.",
+            "type": "string",
+            "enum":[
+                "DCP/2 Analysis",
+                "Contributor",
+                "ArrayExpress",
+                "HCA Release",
+                "GEO",
+                "SCEA",
+                "SCP",
+                "DCP/1 Matrix Service",
+                "LungMAP",
+                "Zenodo",
+                "Publication"
+            ],
+            "user_friendly": "File source",
+            "guidelines": "Should be one of: DCP/2 Analysis, Contributor, ArrayExpress, HCA Release, GEO, SCEA, SCP, DCP/1 Matrix Service, LungMAP, Zenodo, Publication",
+            "example": "GEO; Contributor"
+            }
     }
 }