v2.0 Experiment: Dask Cluster and Processing Improvements (4/4)

.gitignore

@@ -112,3 +112,6 @@ mypy.ini
 
 # mac specific
 .DS_store
+
+# ignore dask worker space
+dask-worker-space/

package.json

@@ -1,6 +1,6 @@
 {
   "name": "vast-pipeline",
-  "version": "1.0.0dev",
+  "version": "2.0.0dev",
   "description": "Vast Pipeline code base for processing and analysing telescope images from the Square Kilometre Pathfinder",
   "main": "gulpfile.js",
   "scripts": {

pyproject.toml

@@ -1,18 +1,18 @@
 [tool.poetry]
 name = "vast-pipeline"
-version = "1.0.0dev"
+version = "2.0.0dev"
 description = "Vast Pipeline code base for processing and analysing telescope images from the Square Kilometre Pathfinder"
 authors = ["The VAST Development Team"]
 license = "MIT"
 
 [tool.poetry.dependencies]
-python = ">=3.8.0,<3.10"
+python = ">=3.10.0,<4"
 astropy = "^5.0"
 astroquery = "^0.4.4"
 bokeh = "2.4.2" # must align with @bokeh/bokehjs version in package.json
 cloudpickle = "^1.5.0"
-dask = {extras = ["dataframe"], version = "^2022.1.0"}
-dill = "^0.3.3"
+dask = {extras = ["complete"], version = "^2023.7.0"}
+dill = "^0"
 Django = "^3.2.13"
 django-crispy-forms = "^1.9.2"
 django-environ = "^0.4.5"
@@ -23,14 +23,14 @@ django-tagulous = "^1.1.0"
 Jinja2 = "^3.0.3"
 networkx = "^2.4"
 numpy = "^1.18.1"
-pandas = "^1.2.0"
+pandas = "^2.0.3"
 psutil = "^5.7.0"
 psycopg2 = "^2.8.4"
-pyarrow = "^7.0"
+pyarrow = "12.0.1"
 scipy = "^1.6.0"
 social-auth-app-django = "^3.1.0"
 social-auth-core = "^3.3.3"
-sqlalchemy = "^1.3.11"
+sqlalchemy = "^2.0.17"
 whitenoise = "^5.2.0"
 gevent = { version = "^21.1.2", optional = true }
 gunicorn = { version = "^20.0.4", optional = true }
@@ -39,8 +39,12 @@ strictyaml = "^1.3.2"
 colorcet = "^2.0.6"
 matplotlib = "^3.5.0"
 holoviews = "^1.14.7"
-datashader = "^0.13.0"
+datashader = "^0"
 channels = "^3.0.4"
+numba = "^0.57.1"
+llvmlite = "^0.40.1"
+django-postgres-copy = "^2.7.3"
+distributed = "^2023.7.0"
 
 [tool.poetry.dev-dependencies]
 mkdocs-material = "^8.2.4"

static/js/datatables-pipeline.js

@@ -1,7 +1,7 @@
 let DEFAULT_DATATABLE_BUTTONS = [
-  {extend: 'colvis', className: 'btn-info btn-sm'},
-  {extend: 'csv', className: 'btn-info btn-sm'},
-  {extend: 'excel', className: 'btn-info btn-sm'},
+  { extend: 'colvis', className: 'btn-info btn-sm' },
+  { extend: 'csv', className: 'btn-info btn-sm' },
+  { extend: 'excel', className: 'btn-info btn-sm' },
 ]
 
 // Formatting function for API
@@ -10,15 +10,15 @@ function obj_formatter(obj) {
     let hrefValue = null;
     if (obj.render.url.hasOwnProperty('nested')) {
       let [prefix, col] = [obj.render.url.prefix, obj.render.url.col];
-      hrefValue = function(data, type, row, meta) {
+      hrefValue = function (data, type, row, meta) {
         // split the col on the . for nested JSON and build the selection
         let sel = row;
         col.split('.').forEach(item => sel = sel[item]);
         return '<a href="' + prefix + sel.id + ' "target="_blank">' + data + '</a>';
       };
     } else {
       let [prefix, col] = [obj.render.url.prefix, obj.render.url.col];
-      hrefValue = function(data, type, row, meta) {
+      hrefValue = function (data, type, row, meta) {
         return '<a href="' + prefix + row.id + ' "target="_blank">' + row[col] + '</a>';
       };
     }
@@ -30,14 +30,14 @@ function obj_formatter(obj) {
       obj.render.float.scale,
       obj.render.float.col
     ];
-    let floatFormat = function(data, type, row, meta) {
-        return (row[col] * scale).toFixed(precision);
+    let floatFormat = function (data, type, row, meta) {
+      return (row[col] * scale).toFixed(precision);
     };
     obj.render = floatFormat;
     return obj;
   } else if (obj.render.hasOwnProperty('contains_sibl')) {
     let col = obj.render.contains_sibl.col;
-    let sibl_bool = function(data, type, row, meta) {
+    let sibl_bool = function (data, type, row, meta) {
       if (row[col] > 0) {
         return true;
       } else {
@@ -66,7 +66,7 @@ function drawExternalResultsTable(id, buttons = DEFAULT_DATATABLE_BUTTONS) {
       "columnDefs": [
         {
           "targets": 0,
-          "render": function( data, type, row, meta) {
+          "render": function (data, type, row, meta) {
             if (row["database"] === "SIMBAD") {
               return '<a href="http://simbad.u-strasbg.fr/simbad/sim-id?Ident=' + row['object_name'] + '" target="_blank">' + row['object_name'] + '</a> (' + row['database'] + ')'
             } else if (row["database"] == "NED") {
@@ -80,13 +80,13 @@ function drawExternalResultsTable(id, buttons = DEFAULT_DATATABLE_BUTTONS) {
         },
         {
           "targets": 1,
-          "render": function ( data, type, row, meta ) {
+          "render": function (data, type, row, meta) {
             return row['separation_arcsec'].toFixed(2);
           }
         },
         {
           "targets": 2,
-          "render": function ( data, type, row, meta ) {
+          "render": function (data, type, row, meta) {
             if (row['otype_long'] !== "" && row['otype'] !== row['otype_long']) {
               return '<abbr title="' + row['otype_long'] + '">' + row['otype'] + '</abbr>';
             } else {
@@ -101,7 +101,7 @@ function drawExternalResultsTable(id, buttons = DEFAULT_DATATABLE_BUTTONS) {
 
 
 // Call the dataTables jQuery plugin
-$(document).ready(function() {
+$(document).ready(function () {
 
   $('[data-toggle="tooltip"]').tooltip();
   let dom = (
@@ -116,12 +116,12 @@ $(document).ready(function() {
     dataConfParsed = JSON.parse(dataConfElement.textContent);
     dataConfList = (Array.isArray(dataConfParsed)) ? dataConfParsed : [dataConfParsed];
   }
-  for (let dataConf of dataConfList){
+  for (let dataConf of dataConfList) {
     let table_id = (dataConfList.length == 1) ? '#dataTable' : '#' + dataConf.table_id;
     if (dataConf.hasOwnProperty('api')) {
       // build conf for server side datatable
       let testFields = dataConf.colsFields;
-      testFields.forEach( function(obj) {
+      testFields.forEach(function (obj) {
         if (obj.hasOwnProperty('render')) {
           obj = obj_formatter(obj)
         }
@@ -134,7 +134,7 @@ $(document).ready(function() {
         language: {
           processing: (
             '<div class="spinner-border" role="status">' +
-              '<span class="sr-only">Loading...</span>' +
+            '<span class="sr-only">Loading...</span>' +
             '</div>'
           )
         },
@@ -151,14 +151,14 @@ $(document).ready(function() {
         columns: dataConf.colsFields,
         order: dataConf.order,
         searchDelay: 2000,
-        dom : dom,
+        dom: dom,
         buttons: DEFAULT_DATATABLE_BUTTONS
       };
       // apply deferLoading config, if supplied
       if (dataConf.hasOwnProperty('deferLoading')) {
         dataTableConf.deferLoading = dataConf.deferLoading;
         // change the message printed in the empty table if deferLoading active
-        dataTableConf.initComplete = function(settings, json) {
+        dataTableConf.initComplete = function (settings, json) {
           $("td.dataTables_empty").text("Submit a query to view results");
         }
       }
@@ -171,9 +171,9 @@ $(document).ready(function() {
       // ...
       // ];
       let dataSet = [];
-      dataConf.dataQuery.forEach( function(obj) {
+      dataConf.dataQuery.forEach(function (obj) {
         let row = [];
-        dataConf.colsFields.forEach(function(elem) {
+        dataConf.colsFields.forEach(function (elem) {
           row.push(obj[elem])
         })
         dataSet.push(row)
@@ -201,106 +201,106 @@ $(document).ready(function() {
           {
             "targets": 1,
             "data": "name",
-            "render": function ( data, type, row, meta ) {
+            "render": function (data, type, row, meta) {
               return '<a href="' + meas_url + row[0] + '"target="_blank">' + row[1] + '</a>';
             }
           },
           {
             "targets": 3,
             "data": "image",
-            "render": function ( data, type, row, meta ) {
+            "render": function (data, type, row, meta) {
               return '<a href="' + img_url + row[19] + '"target="_blank">' + row[3] + '</a>';
             }
           },
           {
             "targets": 4,
             "data": "frequency",
-            "render": function ( data, type, row, meta ) {
+            "render": function (data, type, row, meta) {
               return row[4].toFixed(2);
             }
           },
           {
             "targets": 5,
             "data": "ra",
-            "render": function ( data, type, row, meta ) {
+            "render": function (data, type, row, meta) {
               return row[5].toFixed(4);
             }
           },
           {
             "targets": 6,
             "data": "ra_err",
-            "render": function ( data, type, row, meta ) {
-                return (row[6] * 3600.).toFixed(4);
+            "render": function (data, type, row, meta) {
+              return (row[6] * 3600.).toFixed(4);
             }
           },
           {
             "targets": 7,
             "data": "dec",
-            "render": function ( data, type, row, meta ) {
-                return row[7].toFixed(4);
+            "render": function (data, type, row, meta) {
+              return row[7].toFixed(4);
             }
           },
           {
             "targets": 8,
             "data": "dec_err",
-            "render": function ( data, type, row, meta ) {
-                return (row[8] * 3600.).toFixed(4);
+            "render": function (data, type, row, meta) {
+              return (row[8] * 3600.).toFixed(4);
             }
           },
           {
             "targets": 9,
             "data": "flux_peak",
-            "render": function ( data, type, row, meta ) {
-                return (row[9]).toFixed(3);
+            "render": function (data, type, row, meta) {
+              return (row[9]).toFixed(3);
             }
           },
           {
             "targets": 10,
             "data": "flux_peak_err",
-            "render": function ( data, type, row, meta ) {
-                return (row[10]).toFixed(3);
+            "render": function (data, type, row, meta) {
+              return (row[10]).toFixed(3);
             }
           },
           {
             "targets": 11,
             "data": "flux_peak_isl_ratio",
-            "render": function ( data, type, row, meta ) {
-                return (row[11]).toFixed(2);
+            "render": function (data, type, row, meta) {
+              return (row[11]).toFixed(2);
             }
           },
           {
             "targets": 12,
             "data": "flux_int",
-            "render": function ( data, type, row, meta ) {
-                return (row[12]).toFixed(3);
+            "render": function (data, type, row, meta) {
+              return (row[12]).toFixed(3);
             }
           },
           {
             "targets": 13,
             "data": "flux_int_err",
-            "render": function ( data, type, row, meta ) {
-                return (row[13]).toFixed(3);
+            "render": function (data, type, row, meta) {
+              return (row[13]).toFixed(3);
             }
           },
           {
             "targets": 14,
             "data": "flux_int_isl_ratio",
-            "render": function ( data, type, row, meta ) {
-                return (row[14]).toFixed(2);
+            "render": function (data, type, row, meta) {
+              return (row[14]).toFixed(2);
             }
           },
           {
             "targets": 15,
             "data": "local_rms",
-            "render": function ( data, type, row, meta ) {
-                return (row[15]).toFixed(2);
+            "render": function (data, type, row, meta) {
+              return (row[15]).toFixed(2);
             }
           },
           {
             "targets": 16,
             "data": "snr",
-            "render": function ( data, type, row, meta ) {
-                return (row[16]).toFixed(2);
+            "render": function (data, type, row, meta) {
+              return (row[16]).toFixed(2);
             }
           },
           {
@@ -318,7 +318,7 @@ $(document).ready(function() {
   drawExternalResultsTable('#externalResultsTable');
 
   // Trigger the update search on the datatable
-  $("#catalogSearch").on('click', function(e) {
+  $("#catalogSearch").on('click', function (e) {
     let PipeRun = document.getElementById("runSelect");
     let qry_url = dataConfParsed.api;
     if (PipeRun.value != '') {
@@ -338,7 +338,7 @@ $(document).ready(function() {
       qry_url = qry_url + "&coord=" + encodeURIComponent(coord.value);
     };
     if (unit.value) {
-        qry_url = qry_url + "&radiusunit=" + unit.value
+      qry_url = qry_url + "&radiusunit=" + unit.value
     }
     let avg_flux_type = document.getElementById("aveFluxSelect");
     let avg_flux_min = document.getElementById("avgFluxMinSelect");
@@ -510,7 +510,7 @@ $(document).ready(function() {
       qry_url = qry_url + "&source_selection=" + source_selection.value;
     };
     if (source_selection_type.value) {
-      qry_url = qry_url + "&source_selection_type=" +source_selection_type.value;
+      qry_url = qry_url + "&source_selection_type=" + source_selection_type.value;
     };
     table.ajax.url(qry_url);
     table.ajax.reload();
@@ -522,8 +522,8 @@ $(document).ready(function() {
   });
 
   // Trigger the search reset on the datatable
-  $("#resetSearch").on('click', function(e) {
-    $('#runSelect option').prop('selected', function() {
+  $("#resetSearch").on('click', function (e) {
+    $('#runSelect option').prop('selected', function () {
       return this.defaultSelected
     });
     let inputs = [
@@ -536,7 +536,7 @@ $(document).ready(function() {
       'compactnessMinSelect', 'compactnessMaxSelect', 'objectNameInput', 'MinSnrMinSelect',
       'MinSnrMaxSelect', 'MaxSnrMinSelect', 'MaxSnrMaxSelect', 'fluxMaxMinSelect',
       'fluxMaxMaxSelect', 'sourceSelectionSelect',
-      ];
+    ];
     var input;
     for (input of inputs) {
       document.getElementById(input).value = '';