Deployed 4e3d966 with MkDocs version: 1.5.3

python_api/step/finngen/index.html

@@ -52,12 +52,7 @@
 <span class=normal>66</span>
 <span class=normal>67</span>
 <span class=normal>68</span>
-<span class=normal>69</span>
-<span class=normal>70</span>
-<span class=normal>71</span>
-<span class=normal>72</span>
-<span class=normal>73</span>
-<span class=normal>74</span></pre></div></td><td class=code><div><pre><span></span><code><span class=nd>@dataclass</span>
+<span class=normal>69</span></pre></div></td><td class=code><div><pre><span></span><code><span class=nd>@dataclass</span>
 <span class=k>class</span> <span class=nc>FinnGenStep</span><span class=p>:</span>
 <span class=w>    </span><span class=sd>&quot;&quot;&quot;FinnGen ingestion step.</span>
 
@@ -82,39 +77,34 @@
 
     <span class=k>def</span> <span class=nf>__post_init__</span><span class=p>(</span><span class=bp>self</span><span class=p>:</span> <span class=n>FinnGenStep</span><span class=p>)</span> <span class=o>-&gt;</span> <span class=kc>None</span><span class=p>:</span>
 <span class=w>        </span><span class=sd>&quot;&quot;&quot;Run step.&quot;&quot;&quot;</span>
-        <span class=c1># Read the JSON data from the URL.</span>
+        <span class=c1># Fetch study index.</span>
         <span class=n>json_data</span> <span class=o>=</span> <span class=n>urlopen</span><span class=p>(</span><span class=bp>self</span><span class=o>.</span><span class=n>finngen_phenotype_table_url</span><span class=p>)</span><span class=o>.</span><span class=n>read</span><span class=p>()</span><span class=o>.</span><span class=n>decode</span><span class=p>(</span><span class=s2>&quot;utf-8&quot;</span><span class=p>)</span>
         <span class=n>rdd</span> <span class=o>=</span> <span class=bp>self</span><span class=o>.</span><span class=n>session</span><span class=o>.</span><span class=n>spark</span><span class=o>.</span><span class=n>sparkContext</span><span class=o>.</span><span class=n>parallelize</span><span class=p>([</span><span class=n>json_data</span><span class=p>])</span>
         <span class=n>df</span> <span class=o>=</span> <span class=bp>self</span><span class=o>.</span><span class=n>session</span><span class=o>.</span><span class=n>spark</span><span class=o>.</span><span class=n>read</span><span class=o>.</span><span class=n>json</span><span class=p>(</span><span class=n>rdd</span><span class=p>)</span>
-
-        <span class=c1># Parse the study index data.</span>
-        <span class=n>finngen_studies</span> <span class=o>=</span> <span class=n>FinnGenStudyIndex</span><span class=o>.</span><span class=n>from_source</span><span class=p>(</span>
+        <span class=c1># Process study index.</span>
+        <span class=n>study_index</span> <span class=o>=</span> <span class=n>FinnGenStudyIndex</span><span class=o>.</span><span class=n>from_source</span><span class=p>(</span>
             <span class=n>df</span><span class=p>,</span>
             <span class=bp>self</span><span class=o>.</span><span class=n>finngen_release_prefix</span><span class=p>,</span>
             <span class=bp>self</span><span class=o>.</span><span class=n>finngen_sumstat_url_prefix</span><span class=p>,</span>
             <span class=bp>self</span><span class=o>.</span><span class=n>finngen_sumstat_url_suffix</span><span class=p>,</span>
         <span class=p>)</span>
-
-        <span class=c1># Write the study index output.</span>
-        <span class=n>finngen_studies</span><span class=o>.</span><span class=n>df</span><span class=o>.</span><span class=n>write</span><span class=o>.</span><span class=n>mode</span><span class=p>(</span><span class=bp>self</span><span class=o>.</span><span class=n>session</span><span class=o>.</span><span class=n>write_mode</span><span class=p>)</span><span class=o>.</span><span class=n>parquet</span><span class=p>(</span>
+        <span class=c1># Write study index.</span>
+        <span class=n>study_index</span><span class=o>.</span><span class=n>df</span><span class=o>.</span><span class=n>write</span><span class=o>.</span><span class=n>mode</span><span class=p>(</span><span class=bp>self</span><span class=o>.</span><span class=n>session</span><span class=o>.</span><span class=n>write_mode</span><span class=p>)</span><span class=o>.</span><span class=n>parquet</span><span class=p>(</span>
             <span class=bp>self</span><span class=o>.</span><span class=n>finngen_study_index_out</span>
         <span class=p>)</span>
 
-        <span class=c1># Prepare list of files for ingestion.</span>
-        <span class=n>input_filenames</span> <span class=o>=</span> <span class=p>[</span>
-            <span class=n>row</span><span class=o>.</span><span class=n>summarystatsLocation</span> <span class=k>for</span> <span class=n>row</span> <span class=ow>in</span> <span class=n>finngen_studies</span><span class=o>.</span><span class=n>collect</span><span class=p>()</span>
-        <span class=p>]</span>
+        <span class=c1># Fetch summary stats.</span>
+        <span class=n>input_filenames</span> <span class=o>=</span> <span class=p>[</span><span class=n>row</span><span class=o>.</span><span class=n>summarystatsLocation</span> <span class=k>for</span> <span class=n>row</span> <span class=ow>in</span> <span class=n>study_index</span><span class=o>.</span><span class=n>collect</span><span class=p>()]</span>
         <span class=n>summary_stats_df</span> <span class=o>=</span> <span class=bp>self</span><span class=o>.</span><span class=n>session</span><span class=o>.</span><span class=n>spark</span><span class=o>.</span><span class=n>read</span><span class=o>.</span><span class=n>option</span><span class=p>(</span><span class=s2>&quot;delimiter&quot;</span><span class=p>,</span> <span class=s2>&quot;</span><span class=se>\t</span><span class=s2>&quot;</span><span class=p>)</span><span class=o>.</span><span class=n>csv</span><span class=p>(</span>
             <span class=n>input_filenames</span><span class=p>,</span> <span class=n>header</span><span class=o>=</span><span class=kc>True</span>
         <span class=p>)</span>
-
-        <span class=c1># Specify data processing instructions.</span>
-        <span class=n>summary_stats_df</span> <span class=o>=</span> <span class=n>FinnGenSummaryStats</span><span class=o>.</span><span class=n>from_finngen_harmonized_summary_stats</span><span class=p>(</span>
-            <span class=n>summary_stats_df</span>
-        <span class=p>)</span><span class=o>.</span><span class=n>df</span>
-
-        <span class=c1># Sort and partition for output.</span>
-        <span class=n>summary_stats_df</span><span class=o>.</span><span class=n>sortWithinPartitions</span><span class=p>(</span><span class=s2>&quot;position&quot;</span><span class=p>)</span><span class=o>.</span><span class=n>write</span><span class=o>.</span><span class=n>partitionBy</span><span class=p>(</span>
-            <span class=s2>&quot;studyId&quot;</span><span class=p>,</span> <span class=s2>&quot;chromosome&quot;</span>
-        <span class=p>)</span><span class=o>.</span><span class=n>mode</span><span class=p>(</span><span class=bp>self</span><span class=o>.</span><span class=n>session</span><span class=o>.</span><span class=n>write_mode</span><span class=p>)</span><span class=o>.</span><span class=n>parquet</span><span class=p>(</span><span class=bp>self</span><span class=o>.</span><span class=n>finngen_summary_stats_out</span><span class=p>)</span>
+        <span class=c1># Process summary stats.</span>
+        <span class=n>summary_stats_df</span> <span class=o>=</span> <span class=n>FinnGenSummaryStats</span><span class=o>.</span><span class=n>from_source</span><span class=p>(</span><span class=n>summary_stats_df</span><span class=p>)</span><span class=o>.</span><span class=n>df</span>
+        <span class=c1># Write summary stats.</span>
+        <span class=p>(</span>
+            <span class=n>summary_stats_df</span><span class=o>.</span><span class=n>sortWithinPartitions</span><span class=p>(</span><span class=s2>&quot;position&quot;</span><span class=p>)</span>
+            <span class=o>.</span><span class=n>write</span><span class=o>.</span><span class=n>partitionBy</span><span class=p>(</span><span class=s2>&quot;studyId&quot;</span><span class=p>,</span> <span class=s2>&quot;chromosome&quot;</span><span class=p>)</span>
+            <span class=o>.</span><span class=n>mode</span><span class=p>(</span><span class=bp>self</span><span class=o>.</span><span class=n>session</span><span class=o>.</span><span class=n>write_mode</span><span class=p>)</span>
+            <span class=o>.</span><span class=n>parquet</span><span class=p>(</span><span class=bp>self</span><span class=o>.</span><span class=n>finngen_summary_stats_out</span><span class=p>)</span>
+        <span class=p>)</span>
 </code></pre></div></td></tr></table></div> </details> <div class="doc doc-children"> </div> </div> </div> <hr> <div class=md-source-file> <small> <ul class="metadata page-metadata" data-bi-name="page info" lang=en-us dir=ltr> <!-- mkdocs-git-revision-date-plugin --> <li> <span class="icon twemoji"> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M14.47 15.08 11 13V7h1.5v5.25l3.08 1.83c-.41.28-.79.62-1.11 1m-1.39 4.84c-.36.05-.71.08-1.08.08-4.42 0-8-3.58-8-8s3.58-8 8-8 8 3.58 8 8c0 .37-.03.72-.08 1.08.69.1 1.33.32 1.92.64.1-.56.16-1.13.16-1.72 0-5.5-4.5-10-10-10S2 6.5 2 12s4.47 10 10 10c.59 0 1.16-.06 1.72-.16-.32-.59-.54-1.23-.64-1.92M18 15v3h-3v2h3v3h2v-3h3v-2h-3v-3h-2Z"/></svg> </span> <span class=label> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-timeago"><span class=timeago datetime=2023-05-09T15:08:02+00:00 locale=en></span></span><span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-05-09</span> </span> </li> <li> <span class="icon twemoji"> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M21 13.1c-.1 0-.3.1-.4.2l-1 1 2.1 2.1 1-1c.2-.2.2-.6 0-.8l-1.3-1.3c-.1-.1-.2-.2-.4-.2m-1.9 1.8-6.1 6V23h2.1l6.1-6.1-2.1-2M12.5 7v5.2l4 2.4-1 1L11 13V7h1.5M11 21.9c-5.1-.5-9-4.8-9-9.9C2 6.5 6.5 2 12 2c5.3 0 9.6 4.1 10 9.3-.3-.1-.6-.2-1-.2s-.7.1-1 .2C19.6 7.2 16.2 4 12 4c-4.4 0-8 3.6-8 8 0 4.1 3.1 7.5 7.1 7.9l-.1.2v1.8Z"/></svg> </span> <span class=label> <span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-timeago"><span class=timeago datetime=2023-10-31T12:33:26+00:00 locale=en></span></span><span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-iso_date">2023-10-31</span> </span> </li> <li> <span class="icon twemoji"> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M12 2A10 10 0 0 0 2 12c0 4.42 2.87 8.17 6.84 9.5.5.08.66-.23.66-.5v-1.69c-2.77.6-3.36-1.34-3.36-1.34-.46-1.16-1.11-1.47-1.11-1.47-.91-.62.07-.6.07-.6 1 .07 1.53 1.03 1.53 1.03.87 1.52 2.34 1.07 2.91.83.09-.65.35-1.09.63-1.34-2.22-.25-4.55-1.11-4.55-4.92 0-1.11.38-2 1.03-2.71-.1-.25-.45-1.29.1-2.64 0 0 .84-.27 2.75 1.02.79-.22 1.65-.33 2.5-.33.85 0 1.71.11 2.5.33 1.91-1.29 2.75-1.02 2.75-1.02.55 1.35.2 2.39.1 2.64.65.71 1.03 1.6 1.03 2.71 0 3.82-2.34 4.66-4.57 4.91.36.31.69.92.69 1.85V21c0 .27.16.59.67.5C19.14 20.16 22 16.42 22 12A10 10 0 0 0 12 2Z"/></svg> </span> <span class=label> Contributors </span> </li> <li class=contributors-holder> <ul class=contributors data-bi-name=contributors><li><a href=https://github.com/d0choa title=d0choa data-bi-name=contributorprofile target=_blank><img src=https://avatars.githubusercontent.com/u/5097586 alt=d0choa></a></li><li><a href=https://github.com/tskir title=tskir data-bi-name=contributorprofile target=_blank><img src=https://avatars.githubusercontent.com/u/10669118 alt=tskir></a></li></ul> </li> </ul> </small> </div> </article> </div> </div> <button type=button class="md-top md-icon" data-md-component=top hidden> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12Z"/></svg> Back to top </button> </main> <footer class=md-footer> <div class="md-footer-meta md-typeset"> <div class="md-footer-meta__inner md-grid"> <div class=md-copyright> Made with <a href=https://squidfunk.github.io/mkdocs-material/ target=_blank rel=noopener> Material for MkDocs </a> </div> <div class=md-social> <a href=https://github.com/genetics_etl_python target=_blank rel=noopener title=github.com class=md-social__link> <svg xmlns=http://www.w3.org/2000/svg viewbox="0 0 496 512"><!-- Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2023 Fonticons, Inc.--><path d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"/></svg> </a> </div> </div> </div> </footer> </div> <div class=md-dialog data-md-component=dialog> <div class="md-dialog__inner md-typeset"></div> </div> <script id=__config type=application/json>{"base": "../../..", "features": ["navigation.instant", "navigation.top", "navigation.tabs", "navigation.tabs.sticky", "content.code.annotate", "search.suggest", "search.highlight"], "search": "../../../assets/javascripts/workers/search.f886a092.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script> <script src=../../../assets/javascripts/bundle.aecac24b.min.js></script> <script src=../../../js/timeago.min.js></script> <script src=../../../js/timeago_mkdocs_material.js></script> <script src=../../../assets/javascripts/extra.js></script> </body> </html>