Merge pull request #189 from opentargets/il-geneindex-missing-cols

Add missing columns in gene index
opentargets · Oct 19, 2023 · 6389ff1 · 6389ff1
2 parents e26f2c4 + ec23766
commit 6389ff1
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 24 deletions.
diff --git a/config/datasets/gcp.yaml b/config/datasets/gcp.yaml
@@ -27,7 +27,7 @@ ld_index_raw_template: gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.geno
 ld_matrix_template: gs://gcp-public-data--gnomad/release/2.1.1/ld/gnomad.genomes.r2.1.1.{POP}.common.adj.ld.bm
 
 # Output datasets
-gene_index: ${datasets.outputs}/gene_index/gene_index
+gene_index: ${datasets.outputs}/gene_index
 variant_annotation: ${datasets.outputs}/variant_annotation
 variant_index: ${datasets.outputs}/variant_index
 study_locus: ${datasets.outputs}/study_locus

diff --git a/src/otg/assets/schemas/gene_index.json b/src/otg/assets/schemas/gene_index.json
@@ -32,30 +32,14 @@
       "metadata": {}
     },
     {
+      "metadata": {},
       "name": "obsoleteSymbols",
-      "type": {
-        "type": "array",
-        "elementType": {
-          "type": "struct",
-          "fields": [
-            {
-              "name": "label",
-              "type": "string",
-              "nullable": true,
-              "metadata": {}
-            },
-            {
-              "name": "source",
-              "type": "string",
-              "nullable": true,
-              "metadata": {}
-            }
-          ]
-        },
-        "containsNull": true
-      },
       "nullable": true,
-      "metadata": {}
+      "type": {
+        "containsNull": true,
+        "elementType": "string",
+        "type": "array"
+      }
     },
     {
       "name": "tss",

diff --git a/src/otg/dataset/gene_index.py b/src/otg/dataset/gene_index.py
@@ -64,7 +64,7 @@ def symbols_lut(self: GeneIndex) -> DataFrame:
         """
         return self.df.select(
             f.explode(
-                f.array_union(f.array("approvedSymbol"), f.col("obsoleteSymbols.label"))
+                f.array_union(f.array("approvedSymbol"), f.col("obsoleteSymbols"))
             ).alias("geneSymbol"),
             "*",
         )
diff --git a/src/otg/datasource/open_targets/target.py b/src/otg/datasource/open_targets/target.py
@@ -49,6 +49,10 @@ def as_gene_index(cls: type[GeneIndex], target_index: DataFrame) -> GeneIndex:
         return GeneIndex(
             _df=target_index.select(
                 f.coalesce(f.col("id"), f.lit("unknown")).alias("geneId"),
+                "approvedSymbol",
+                "approvedName",
+                "biotype",
+                f.col("obsoleteSymbols.label").alias("obsoleteSymbols"),
                 f.coalesce(f.col("genomicLocation.chromosome"), f.lit("unknown")).alias(
                     "chromosome"
                 ),