From b686ef008ab811c6f8c72cfefb38ed4f02f95062 Mon Sep 17 00:00:00 2001 From: Ryan Bunney Date: Fri, 27 Dec 2024 13:28:02 +0800 Subject: [PATCH] LIU-439: Use translator avahi approach for engine. Also added quickstart for README.rst to help people get started ASAP, based on user feedback. --- README.rst | 52 ++++++++++++++++++++++--- daliuge-common/docker/Dockerfile.dev | 6 +-- daliuge-engine/docker/Dockerfile.dev | 6 ++- daliuge-engine/docker/Dockerfile.devall | 9 ++++- daliuge-engine/docker/Dockerfile.ray | 6 ++- 5 files changed, 67 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 67b72c228..8dd1087ac 100644 --- a/README.rst +++ b/README.rst @@ -33,17 +33,59 @@ It originated from a prototyping activity as part of the `SDP Consortium `_ called Data Flow Management System (DFMS). DFMS aimed to prototype the execution framework of the proposed SDP architecture. -For more information about the installation and usage of the system please refer to the `documentation `_ - - Development and maintenance of |daliuge| is currently hosted at ICRAR_ and is performed by the `DIA team `_. -See the ``docs/`` directory for more information, or visit `our online -documentation `_ +Quickstart +========== + +Most users will need to use DALiuGE locally only when prototyping their workflow as it +is designed in EAGLE. This repository provides a `Makefile` that simplifies the setup +options for running DALiuGE locally, without needing to take into account the distributed +and flexible nature of the ecosystem. + +.. note:: It is recommended to have Docker installed when running DALiuGE as a workflow +prototyping tool. Please review the installation material for Docker `here `. + +The following steps are recommended for quickstarting your DALiuGE install: + +1. Create and enterthe virtual environment +2. Build the docker images +3. Run the docker images +4. Confirm the images are running and accessible from EAGLE using a web browser. + +Creating the virtual environment is as simple as running:: + + make virtualenv + source .venv/bin/activate + +Now, in the ``.venv`` environment, run:: + make docker-install +This will install a development version of the DALiuGE images and is appropriate for local +installation (*not* a production environment). +Running both the Engine and the Translator is as a simple as:: + + make docker-run + +It is possible to confirm the running of both by accessing the following links in a browser +(Opera, Firefox, or Chrome are recommended):: + + http://dlg-trans.local:8084/ # Translator + http://dlg-engine.local:8000/ # Node Manager + http://dlg-engine.local:8001/ # Data Island Manager + + +With this setup running, it is now possible to translate and deploy a prototype EAGLE workflow +on your local machine. + +For more information about the installation and usage of the system please refer to the `documentation `_ + + +See the ``docs/`` directory for more information, or visit `our online +documentation `_ .. |daliuge| replace:: DALiuGE diff --git a/daliuge-common/docker/Dockerfile.dev b/daliuge-common/docker/Dockerfile.dev index f3a6589fe..e5b0376bf 100644 --- a/daliuge-common/docker/Dockerfile.dev +++ b/daliuge-common/docker/Dockerfile.dev @@ -9,15 +9,15 @@ LABEL stage=builder LABEL build=$BUILD_ID RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone RUN apt-get update && \ - apt-get install -y avahi-utils gcc python3 python3.8-venv python3-pip python3-distutils python3-appdirs libmetis-dev curl git sudo && \ + apt-get install -y avahi-daemon avahi-utils gcc python3.9 python3.9-venv python3-pip python3-distutils python3-appdirs libmetis-dev curl git sudo && \ apt-get clean COPY / /daliuge -RUN cd / && python3 -m venv dlg && cd /daliuge && \ +RUN cd / && python3.9 -m venv dlg && cd /daliuge && \ . /dlg/bin/activate && \ pip install --upgrade pip && \ pip install wheel numpy && \ pip install . -# we don't clean this up, will be done in the derived images \ No newline at end of file +# we don't clean this up, will be done in the derived images diff --git a/daliuge-engine/docker/Dockerfile.dev b/daliuge-engine/docker/Dockerfile.dev index 401261788..f226bd41c 100644 --- a/daliuge-engine/docker/Dockerfile.dev +++ b/daliuge-engine/docker/Dockerfile.dev @@ -4,7 +4,11 @@ FROM icrar/daliuge-common:${VCS_TAG:-latest} # RUN sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata \ # gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev -RUN service dbus start && service avahi-daemon start && avahi-set-host-name dlg-engine +RUN service avahi-daemon stop && \ + service dbus start && \ + service avahi-daemon start && \ + avahi-set-host-name dlg-engine + COPY / /daliuge RUN . /dlg/bin/activate && pip install --upgrade pip && pip install wheel && cd /daliuge && \ pip install . diff --git a/daliuge-engine/docker/Dockerfile.devall b/daliuge-engine/docker/Dockerfile.devall index c93b3c246..f29d83120 100644 --- a/daliuge-engine/docker/Dockerfile.devall +++ b/daliuge-engine/docker/Dockerfile.devall @@ -5,10 +5,16 @@ FROM icrar/daliuge-common:${VCS_TAG:-latest} # RUN sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata \ # gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev +RUN apt install -y git python3.9-dev + RUN apt-get update &&\ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc curl python3-pip python3-numpy -RUN service dbus start && service avahi-daemon start && avahi-set-host-name dlg-engine +RUN service avahi-daemon stop && \ + service dbus start && \ + service avahi-daemon start && \ + avahi-set-host-name dlg-engine + # USER ${USER} COPY / /daliuge RUN . /dlg/bin/activate && pip install wheel && cd /daliuge && \ @@ -26,7 +32,6 @@ ENV VIRTUAL_ENV=/dlg ENV PATH="$VIRTUAL_ENV/bin:$PATH" ENV DLG_ROOT="/tmp/dlg" -RUN apt install -y git python3-dev # Nifty #RUN pip install --prefix=$PYTHON_PREFIX git+https://gitlab.com/ska-telescope/sdp/ska-gridder-nifty-cuda.git diff --git a/daliuge-engine/docker/Dockerfile.ray b/daliuge-engine/docker/Dockerfile.ray index 390acbc2f..ef7f62be1 100644 --- a/daliuge-engine/docker/Dockerfile.ray +++ b/daliuge-engine/docker/Dockerfile.ray @@ -6,7 +6,11 @@ FROM icrar/daliuge-common:${VCS_TAG:-latest} # gcc g++ gdb casacore-dev clang-tidy-10 clang-tidy libboost1.71-all-dev libgsl-dev RUN sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends gcc python3-pip curl -RUN service dbus start && service avahi-daemon start && avahi-set-host-name dlg-engine + +RUN service avahi-daemon stop && \ + service dbus start && \ + service avahi-daemon start && \ + avahi-set-host-name dlg-engine COPY / /daliuge RUN . /home/ray/dlg/bin/activate && \