diff --git a/dlg-graphs/B_process_each_source_2.graph b/dlg-graphs/B_process_each_source_2.graph new file mode 100644 index 00000000..f737ce79 --- /dev/null +++ b/dlg-graphs/B_process_each_source_2.graph @@ -0,0 +1,870 @@ +{ + "modelData": { + "filePath": "dlg-graphs/B_process_each_source_2.graph", + "fileType": "Graph", + "shortDescription": "", + "detailedDescription": "", + "repoService": "GitHub", + "repoBranch": "main", + "repo": "ICRAR/wallaby-hires", + "eagleVersion": "v6.2.0:html-graph-renderer", + "eagleCommitHash": "a1d68d87110d9cf6614ceb5e9f6b807ffad4498f", + "schemaVersion": "OJS", + "readonly": true, + "repositoryUrl": "", + "commitHash": "", + "downloadUrl": "", + "signature": "", + "lastModifiedName": "", + "lastModifiedEmail": "", + "lastModifiedDatetime": 0, + "numLGNodes": 5 + }, + "nodeDataArray": [ + { + "category": "Scatter", + "categoryType": "Construct", + "isGroup": true, + "color": "rgb(227 189 100)", + "drawOrderHint": 0, + "key": -1, + "name": "each_source", + "description": "For every line in the catalogue (.csv) file. ", + "x": 949.3731541316781, + "y": 394.96860688770346, + "radius": 296.8236123863249, + "collapsed": false, + "subject": null, + "expanded": true, + "repositoryUrl": "", + "commitHash": "", + "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master.palette", + "dataHash": "89328904851744addae66eedd9288539faf704b1c19c729763c1158566b7d24e", + "fields": [ + { + "name": "num_of_copies", + "value": 1, + "defaultValue": "1", + "description": "Specifies the number of replications that will be generated of the scatter construct's contents.", + "readonly": false, + "type": "Integer", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "ba8a51cf-b6f2-4f08-9561-374050bb4e46", + "parameterType": "ConstructParameter", + "usage": "NoPort" + }, + { + "name": "dropclass", + "value": "dlg.apps.constructs.ScatterDrop", + "defaultValue": "dlg.apps.constructs.ScatterDrop", + "description": "Drop class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "bd02e277-0018-4808-bd59-73ff644f34a8", + "parameterType": "ComponentParameter", + "usage": "NoPort" + } + ], + "inputAppFields": [ + { + "name": "dropclass", + "value": "dlg.apps.simple.PickOne", + "defaultValue": "dlg.apps.simple.PickOne", + "description": "Application class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "9120a87a-72f5-4c24-9588-ae3ec445bf3c", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "input_parser", + "value": "pickle", + "defaultValue": "pickle", + "description": "Input port parsing technique", + "readonly": false, + "type": "Select", + "precious": false, + "options": [ + "raw", + "pickle", + "eval", + "npy", + "path", + "dataurl" + ], + "positional": false, + "keyAttribute": false, + "id": "d4f7a76c-64b0-410b-ad0c-b23920a9d2b7", + "parameterType": "ApplicationArgument", + "usage": "NoPort" + }, + { + "name": "output_parser", + "value": "pickle", + "defaultValue": "pickle", + "description": "Output port parsing technique", + "readonly": false, + "type": "Select", + "precious": false, + "options": [ + "raw", + "pickle", + "eval", + "npy", + "path", + "dataurl" + ], + "positional": false, + "keyAttribute": false, + "id": "1f2caff8-360e-4533-891c-69329a69a95f", + "parameterType": "ApplicationArgument", + "usage": "NoPort" + }, + { + "name": "execution_time", + "value": 5, + "defaultValue": "5", + "description": "Estimated execution time", + "readonly": false, + "type": "Float", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "5fc67993-b79b-4009-ba23-a3314b706dfc", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "num_cpus", + "value": 1, + "defaultValue": "1", + "description": "Number of cores used", + "readonly": false, + "type": "Integer", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "80b0609f-606f-4d7e-b108-e69e006644ba", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "group_start", + "value": false, + "defaultValue": "False", + "description": "Is this node the start of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "c0fb94de-05f8-49d6-9547-7f53a21ceb00", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "list_of_sources", + "value": "", + "defaultValue": "", + "description": "List of elements", + "readonly": false, + "type": "Object.array", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "f9aeedd2-03fd-44f6-bb9f-072f4079caf2", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + }, + { + "name": "source", + "value": "", + "defaultValue": "", + "description": "Port carrying the first element of input array", + "readonly": false, + "type": "Object.element", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "f1ce3daf-bed6-43ec-8818-680fe7094692", + "parameterType": "ApplicationArgument", + "usage": "OutputPort" + } + ], + "outputAppFields": [], + "inputApplicationName": "PickOne", + "inputApplicationType": "PythonApp", + "inputApplicationKey": -3, + "inputApplicationDescription": "App that picks the first element of an input list, passes that to all outputs, except the first one. The first output is used to pass the remaining array on. This app is useful for a loop.", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + }, + { + "category": "PythonApp", + "categoryType": "Application", + "isGroup": false, + "color": "#0059a5", + "drawOrderHint": 0, + "key": -22, + "name": "process_each_source", + "description": "For each line (i.e. for every source) in the .csv file:\n- By splitting the comma-separated values. ", + "x": 974.4257795966305, + "y": 393.2655958339034, + "radius": 25, + "collapsed": true, + "subject": null, + "expanded": true, + "repositoryUrl": "https://github.com/ICRAR/daliuge", + "commitHash": "44edda3", + "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master-template.palette", + "dataHash": "deea3124ecb065724939d4702d8ae3b44cddcfc40e806c5cf5cf4d83c38e9580", + "group": -1, + "fields": [ + { + "name": "func_name", + "value": "", + "defaultValue": "", + "description": "Python function name", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "cb3b380f-d348-4871-b424-ae7e248e9ef7", + "parameterType": "ApplicationArgument", + "usage": "NoPort" + }, + { + "name": "func_code", + "value": "", + "defaultValue": "", + "description": "Python function code, e.g. 'def function_name(args): return args'", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "8dbefb1f-3746-4657-afe5-2e5bf22f242c", + "parameterType": "ApplicationArgument", + "usage": "NoPort" + }, + { + "name": "dropclass", + "value": "dlg.apps.pyfunc.PyFuncApp", + "defaultValue": "dlg.apps.pyfunc.PyFuncApp", + "description": "Application class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "46727ea6-3b5e-4ebe-8755-5d0ab2f5dfb3", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "execution_time", + "value": 5, + "defaultValue": "5", + "description": "Estimated execution time", + "readonly": false, + "type": "Float", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "593bc5f4-f4a4-44e0-99f0-f0e653e323e6", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "num_cpus", + "value": 1, + "defaultValue": "1", + "description": "Number of cores used", + "readonly": false, + "type": "Integer", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "7f4d5f32-e7c7-4949-8041-3bd7c0ab50de", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "group_start", + "value": false, + "defaultValue": "False", + "description": "Is this node the start of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "9d5a8a5f-fc69-4ee5-b805-ed9e1bf72a4f", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "input_error_threshold", + "value": 0, + "defaultValue": "0", + "description": "The allowed failure rate of the inputs (in percent), before this component goes to ERROR state and is not executed", + "readonly": false, + "type": "Integer", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "c2ad6d97-e89c-4a64-8cc9-106726fc5d49", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "n_tries", + "value": 1, + "defaultValue": "1", + "description": "Specifies the number of times the 'run' method will be executed before finally giving up", + "readonly": false, + "type": "Integer", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "bec4cbe5-3289-4900-a2d8-be98876a29be", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "input_parser", + "value": "pickle", + "defaultValue": "pickle", + "description": "Input port parsing technique", + "readonly": false, + "type": "Object", + "precious": false, + "options": [ + "raw", + "pickle", + "eval", + "npy", + "path", + "dataurl" + ], + "positional": false, + "keyAttribute": true, + "id": "b3a6bc47-e690-45a3-95a4-4a5f703bad5e", + "parameterType": "ApplicationArgument", + "usage": "InputPort" + }, + { + "name": "output_parser", + "value": "pickle", + "defaultValue": "pickle", + "description": "Output port parsing technique Mapping from argname to default value. Should match only the last part of the argnames list. Values are interpreted as Python code literals and that means string values need to be quoted.", + "readonly": false, + "type": "Object", + "precious": false, + "options": [ + "raw", + "pickle", + "eval", + "npy", + "path", + "dataurl" + ], + "positional": false, + "keyAttribute": true, + "id": "4f5078c6-afc9-4ed1-bc74-75191545bb9a", + "parameterType": "ApplicationArgument", + "usage": "OutputPort" + }, + { + "name": "func_arg_mapping", + "value": "", + "defaultValue": "", + "description": " Mapping between argument name and input drop uids", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "db158c5f-9b83-4268-b4ea-17eaab15ad91", + "parameterType": "ApplicationArgument", + "usage": "NoPort" + } + ], + "inputAppFields": [], + "outputAppFields": [], + "inputApplicationName": "", + "inputApplicationType": "None", + "inputApplicationKey": null, + "inputApplicationDescription": "", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + }, + { + "category": "Memory", + "categoryType": "Data", + "isGroup": false, + "color": "#2c2c2c", + "drawOrderHint": 0, + "key": -27, + "name": "source", + "description": "In-memory storage of intermediate data products", + "x": 826.5881504946165, + "y": 395.0818941474958, + "radius": 25, + "collapsed": true, + "subject": null, + "expanded": true, + "repositoryUrl": "https://github.com/ICRAR/daliuge", + "commitHash": "44edda3", + "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master.palette", + "dataHash": "99562ee043ee082bb2633032ff1c7970a2ea33053d30d6fd46d4a596c5deab69", + "group": -1, + "fields": [ + { + "name": "dropclass", + "value": "dlg.data.drops.memory.InMemoryDROP", + "defaultValue": "dlg.data.drops.memory.InMemoryDROP", + "description": "Drop class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "5fa7376c-7692-4461-8bd9-a5b9d642da3b", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "data_volume", + "value": 5, + "defaultValue": "5", + "description": "Estimated size of the data contained in this node", + "readonly": false, + "type": "Float", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "a2316b6a-6e2f-4624-ae2a-1604b0465f23", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "group_end", + "value": false, + "defaultValue": "False", + "description": "Is this node the end of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "19a2b830-c28b-4f87-b7c4-90b63030419c", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "streaming", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component streams input and output data", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "7c207450-87f8-408c-8b48-2e5b9448cdbf", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "persist", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component contains data that should not be deleted after execution", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "b1f4ca53-5003-4610-8de0-bab78c656e72", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "dummy", + "value": "", + "defaultValue": "", + "description": "Dummy port", + "readonly": false, + "type": "Object", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "56104deb-e2d7-4fa7-a132-fabcd743c3ad", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + }, + { + "name": "element", + "value": "", + "defaultValue": "", + "description": "", + "readonly": false, + "type": "Object.element", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "c9e6dbce-809f-47e4-bb7a-0e4beaec8669", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + } + ], + "inputAppFields": [], + "outputAppFields": [], + "inputApplicationName": "", + "inputApplicationType": "None", + "inputApplicationKey": null, + "inputApplicationDescription": "", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + }, + { + "category": "Memory", + "categoryType": "Data", + "isGroup": false, + "color": "#2c2c2c", + "drawOrderHint": 0, + "key": -28, + "name": "Dictionary", + "description": "For each source, we have the following values: {name, RA, DEC, System Velocity}", + "x": 1191.1896887222927, + "y": 396.8187707068335, + "radius": 25, + "collapsed": true, + "subject": null, + "expanded": true, + "repositoryUrl": "https://github.com/ICRAR/daliuge", + "commitHash": "44edda3", + "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master.palette", + "dataHash": "99562ee043ee082bb2633032ff1c7970a2ea33053d30d6fd46d4a596c5deab69", + "group": -1, + "fields": [ + { + "name": "dropclass", + "value": "dlg.data.drops.memory.InMemoryDROP", + "defaultValue": "dlg.data.drops.memory.InMemoryDROP", + "description": "Drop class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "96388b61-5e03-4acc-b31d-88c285e25ebf", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "data_volume", + "value": 5, + "defaultValue": "5", + "description": "Estimated size of the data contained in this node", + "readonly": false, + "type": "Float", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "bf3b0b9b-6d48-4732-889f-a2b070359e50", + "parameterType": "ConstraintParameter", + "usage": "NoPort" + }, + { + "name": "group_end", + "value": false, + "defaultValue": "False", + "description": "Is this node the end of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "2595789b-fe4a-4df3-aacf-d72db284a611", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "streaming", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component streams input and output data", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "5072d871-0a58-4abc-99a7-caa5a31c6322", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "persist", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component contains data that should not be deleted after execution", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "bd8f3f2e-587c-4782-ba37-a22cfaad6433", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "dummy", + "value": "", + "defaultValue": "", + "description": "Dummy port", + "readonly": false, + "type": "Object", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "168a73de-2857-411b-aba1-93d16c52ff4a", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + }, + { + "name": "Output", + "value": "", + "defaultValue": "", + "description": "", + "readonly": false, + "type": "Object", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "397026a1-21d1-4deb-bdf4-04d98ebbd902", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + } + ], + "inputAppFields": [], + "outputAppFields": [], + "inputApplicationName": "", + "inputApplicationType": "None", + "inputApplicationKey": null, + "inputApplicationDescription": "", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + }, + { + "category": "Memory", + "categoryType": "Data", + "isGroup": false, + "color": "#2c2c2c", + "drawOrderHint": 0, + "key": -2, + "name": "source_list", + "description": "In-memory storage of intermediate data products. \n- List of lists. \n- Format: [ ['Source1', '05h30m00s', '40d30m00s', 100], [...], [...], [...], [...] ]", + "x": 485.9627741087308, + "y": 392.7039303606033, + "radius": 25, + "collapsed": true, + "subject": null, + "expanded": true, + "repositoryUrl": "https://github.com/ICRAR/daliuge", + "commitHash": "44edda3", + "paletteDownloadUrl": "https://raw.githubusercontent.com/ICRAR/EAGLE_test_repo/master/daliuge/daliuge-master.palette", + "dataHash": "99562ee043ee082bb2633032ff1c7970a2ea33053d30d6fd46d4a596c5deab69", + "fields": [ + { + "name": "dropclass", + "value": "dlg.data.drops.memory.InMemoryDROP", + "defaultValue": "dlg.data.drops.memory.InMemoryDROP", + "description": "Drop class", + "readonly": false, + "type": "String", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "1f1f0ebe-e886-4791-8c97-16de2e9a35b4", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "data_volume", + "value": 5, + "defaultValue": "5", + "description": "Estimated size of the data contained in this node", + "readonly": false, + "type": "Float", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "e39a0f71-2b81-429f-8066-3c37a5599146", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "group_end", + "value": false, + "defaultValue": "False", + "description": "Is this node the end of a group?", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "672eb504-d357-439d-9691-281ebd8d0872", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "streaming", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component streams input and output data", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "4f644c45-0118-40c9-abab-dd4dc170f39c", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "persist", + "value": false, + "defaultValue": "False", + "description": "Specifies whether this data component contains data that should not be deleted after execution", + "readonly": false, + "type": "Boolean", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "e908cc81-c8fa-4308-942a-40b41e6a2055", + "parameterType": "ComponentParameter", + "usage": "NoPort" + }, + { + "name": "array", + "value": "", + "defaultValue": "", + "description": "", + "readonly": false, + "type": "Object", + "precious": false, + "options": [], + "positional": false, + "keyAttribute": false, + "id": "faf822c0-15a8-49a7-9426-6993678d4d9c", + "parameterType": "ApplicationArgument", + "usage": "InputOutput" + } + ], + "inputAppFields": [], + "outputAppFields": [], + "inputApplicationName": "", + "inputApplicationType": "None", + "inputApplicationKey": null, + "inputApplicationDescription": "", + "outputApplicationName": "", + "outputApplicationType": "None", + "outputApplicationKey": null, + "outputApplicationDescription": "" + } + ], + "linkDataArray": [ + { + "from": -1, + "fromPort": "f1ce3daf-bed6-43ec-8818-680fe7094692", + "to": -27, + "toPort": "c9e6dbce-809f-47e4-bb7a-0e4beaec8669", + "dataType": "Object.element", + "loop_aware": "0", + "closesLoop": false + }, + { + "from": -27, + "fromPort": "c9e6dbce-809f-47e4-bb7a-0e4beaec8669", + "to": -22, + "toPort": "b3a6bc47-e690-45a3-95a4-4a5f703bad5e", + "dataType": "Object.element", + "loop_aware": "0", + "closesLoop": false + }, + { + "from": -22, + "fromPort": "4f5078c6-afc9-4ed1-bc74-75191545bb9a", + "to": -28, + "toPort": "168a73de-2857-411b-aba1-93d16c52ff4a", + "dataType": "Object", + "loop_aware": "0", + "closesLoop": false + }, + { + "from": -2, + "fromPort": "faf822c0-15a8-49a7-9426-6993678d4d9c", + "to": -1, + "toPort": "f9aeedd2-03fd-44f6-bb9f-072f4079caf2", + "dataType": "Object", + "loop_aware": "0", + "closesLoop": false + } + ] +} \ No newline at end of file