diff --git a/.github/workflows/deployment_uat.yml b/.github/workflows/deployment_uat.yml new file mode 100644 index 00000000..3fd1c56a --- /dev/null +++ b/.github/workflows/deployment_uat.yml @@ -0,0 +1,35 @@ +on: + workflow_dispatch: + push: + branches: + - uat +jobs: + build: + name: Build and upload Docker image + runs-on: [self-hosted] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push docker image to Amazon ECR + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ secrets.AWS_ECR_REPOSITORY }} + IMAGE_TAG: ${{ github.sha }} + UAT_TAG: uat + run: | + docker build --build-arg GIT_SHA=${{ github.sha }} -t $REGISTRY/$REPOSITORY:$IMAGE_TAG -t $REGISTRY/$REPOSITORY:$UAT_TAG . + docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG + docker push $REGISTRY/$REPOSITORY:$UAT_TAG diff --git a/.gitignore b/.gitignore index d5f1f922..38f1738b 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,6 @@ reporting-grofwild/inst/extdata/spatialData*.RData # Local copy of data files /dataS3 /data + +# AWS connection +/scripts/putSecret.sh diff --git a/Dockerfile b/Dockerfile index b91bd3b4..1745fccd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,63 +1,63 @@ -FROM openanalytics/r-ver:4.0.5 +FROM rocker/r-ver:4.3.2 LABEL maintainer="Machteld Varewyck machteld.varewyck@openanalytics.eu" -RUN apt-get update && apt-get install -y --no-install-recommends \ - gdal-bin \ - libproj15 \ - libgeos-3.8.0 libgeos-c1v5 \ +RUN apt-get update && apt-get install --no-install-recommends -y \ + libgdal-dev \ + libproj22 \ + libgeos3.10.2 libgeos-c1v5 \ libcurl4-openssl-dev \ curl \ ca-certificates \ pandoc \ - libudunits2-0 \ + libudunits2-dev \ libmagick++-dev \ libssl-dev \ lmodern \ fonts-cantarell \ + texlive-latex-extra \ + texlive-xetex \ + texlive-fonts-recommended \ texlive-plain-generic \ - texlive-fonts-extra \ - && rm -rf /var/lib/apt/lists/* + lmodern \ + wget && \ + wget https://downloads.vivaldi.com/stable/vivaldi-stable_6.8.3381.55-1_$(dpkg --print-architecture).deb && \ + apt-get install --no-install-recommends -y ./vivaldi-stable*.deb && \ + rm -rf /var/lib/apt/lists/* # Use the remotes package instead of devtools as it is much lighter RUN R -q -e "install.packages('remotes')" -RUN R -q -e "remotes::install_cran(c('shiny', 'sf', 'dplyr', 'plyr', 'reshape2', 'mgcv', 'stringr', 'leaflet', 'flexdashboard', 'testthat', 'shinyjs', 'data.table', 'tinytex', 'tidyr', 'arrow', 'kableExtra'))" -RUN R -q -e "remotes::install_version('DT', version = '0.23', repos = 'https://cloud.r-project.org', upgrade = 'never')" -RUN R -q -e "remotes::install_version('plotly', version = '4.10.1', repos = 'https://cloud.r-project.org', upgrade = 'never')" -RUN R -q -e "remotes::install_version('rmarkdown', version = '2.18', repos = 'https://cloud.r-project.org', upgrade = 'never')" -RUN R -q -e "remotes::install_version('magick', version = '2.7.3', repos = 'https://cloud.r-project.org', upgrade = 'never')" +RUN R -q -e "options(warn = 2); remotes::install_cran(c('shiny', 'sf', 'dplyr', 'plyr', 'reshape2', 'mgcv', 'stringr', 'leaflet', 'flexdashboard', 'testthat', 'shinyjs', 'data.table', 'tinytex', 'tidyr', 'arrow', 'kableExtra', 'webshot2'))" +RUN R -q -e "options(warn = 2); remotes::install_version('DT', version = '0.23', repos = 'https://cloud.r-project.org', upgrade = 'never')" +RUN R -q -e "options(warn = 2); remotes::install_version('plotly', version = '4.10.1', repos = 'https://cloud.r-project.org', upgrade = 'never')" +RUN R -q -e "options(warn = 2); remotes::install_version('rmarkdown', version = '2.18', repos = 'https://cloud.r-project.org', upgrade = 'never')" +RUN R -q -e "options(warn = 2); remotes::install_version('magick', version = '2.7.3', repos = 'https://cloud.r-project.org', upgrade = 'never')" # NOTE: Need at least these versions of plotly, rmarkdown and magick for dashboard rmarkdown to work -RUN R -q -e "remotes::install_github('inbo/INBOtheme@v0.5.10')" -RUN R -q -e "install.packages('oaStyle', repos = c(rdepot = 'https://repos.openanalytics.eu/repo/public', getOption('repos')))" - -# to prevent bobbing with shinycssloaders -RUN R -q -e "remotes::install_github('daattali/shinycssloaders')" +RUN R -q -e "options(warn = 2); remotes::install_github(c('inbo/INBOtheme@v0.5.10', 'daattali/shinycssloaders'))" # For the rmarkdown pdf report -RUN R -e "tinytex::install_tinytex()" -ENV PATH="/root/bin:${PATH}" -# many packages needed for kableExtra background feature -RUN R -e "tinytex::tlmgr_install(pkgs = c('fancyhdr', 'sectsty', 'titling', 'grffile', 'texlive-scripts', 'mathspec', 'multirow', 'wrapfig', 'colortbl', 'pdflscape', 'tabu', 'varwidth', 'threeparttable', 'threeparttablex', 'environ', 'trimspaces', 'ulem', 'makecell'))" +RUN R -q -e "options(warn = 2); install.packages('oaStyle', repos = c(rdepot = 'https://repos.openanalytics.eu/repo/public', getOption('repos')))" +#RUN R -e "tinytex::install_tinytex()" +#ENV PATH="/root/bin:${PATH}" +## many packages needed for kableExtra background feature +#RUN R -e "tinytex::tlmgr_install(pkgs = c('fancyhdr', 'sectsty', 'titling', 'grffile', 'texlive-scripts', 'mathspec', 'multirow', 'wrapfig', 'colortbl', 'pdflscape', 'tabu', 'varwidth', 'threeparttable', 'threeparttablex', 'environ', 'trimspaces', 'ulem', 'makecell'))" -# For downloading the maps -# Attention: do not install phantomjs directly, will not work then! -RUN R -q -e "remotes::install_cran('webshot')" -RUN R -e "webshot::install_phantomjs()" +# Configure browser for using webshot2 +ENV CHROMOTE_CHROME=/usr/bin/vivaldi # Git sha ARG GIT_SHA ENV GIT_SHA=$GIT_SHA # For access to S3 on UAT -# For calculating areas - fix #435 -RUN R -q -e "remotes::install_cran(c('config', 'aws.s3', 'aws.ec2metadata', 'lwgeom'))" +RUN R -q -e "options(warn = 2); remotes::install_cran(c('config', 'aws.s3', 'aws.ec2metadata', 'aws.signature'))" # Install the package without the source files ending up in the Docker image COPY reporting-grofwild /tmp/package -RUN R -q -e "remotes::install_local('/tmp/package', dependencies=FALSE)" +RUN R -q -e "options(warn = 2); remotes::install_local('/tmp/package', dependencies=FALSE)" # set host COPY Rprofile.site /usr/local/lib/R/etc/ diff --git a/README.md b/README.md index 3512cbf9..ccac556d 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Configure connection to [S3 data buckets](https://docs.aws.amazon.com/cli/latest Run docker image, public app. ``` -sudo docker run -p 3001:3838 inbo/wildapp R -e "Sys.setenv('AWS_DEFAULT_REGION'='eu-west-1','AWS_ACCESS_KEY_ID'='xxx','AWS_SECRET_ACCESS_KEY'='xxx'); reportingGrofwild::runWildApp(public=TRUE)" +sudo docker run -it -v ~/.aws:/root/.aws -p 3001:3838 inbo/wildapp R -e "reportingGrofwild::setupS3(); reportingGrofwild::runWildApp(public=TRUE)" ``` Browse to `localhost:3001`. @@ -40,13 +40,13 @@ Browse to `localhost:3001`. Run docker image, private app for specific KBO. ``` -sudo docker run -p 3001:3838 inbo/wildapp R -e "Sys.setenv('AWS_DEFAULT_REGION'='eu-west-1','AWS_ACCESS_KEY_ID'='xxx','AWS_SECRET_ACCESS_KEY'='xxx'); reportingGrofwild::runWildApp(public=FALSE, kbo = xxx)" +sudo docker run -it -v ~/.aws:/root/.aws -p 3001:3838 inbo/wildapp R -e "reportingGrofwild::setupS3(); reportingGrofwild::runWildApp(public=FALSE, kbo = xxx)" ``` In a similar way, an R session can be started to run specific functions of the reportingGrofwild R package. ``` -sudo docker run -p 3001:3838 inbo/wildapp R +sudo docker run -it -p 3001:3838 inbo/wildapp R ``` diff --git a/reporting-grofwild/DESCRIPTION b/reporting-grofwild/DESCRIPTION index 386343b7..76506d12 100644 --- a/reporting-grofwild/DESCRIPTION +++ b/reporting-grofwild/DESCRIPTION @@ -5,12 +5,13 @@ Title: Grof Wild analysis for INBO Type: Package Author: Machteld Varewyck, Laure Cougnaud, Eva Adriaensen Description: Data visualization of grof wild in Vlaanderen. -Version: 0.3.5 +Version: 0.3.6 Date: 2024-06-03 Imports: arrow, aws.ec2metadata, aws.s3, + aws.signature, config, data.table, DT, @@ -19,7 +20,7 @@ Imports: htmlwidgets, INBOtheme, leaflet, - methods, + methods, mgcv, plotly, plyr, @@ -33,10 +34,9 @@ Imports: stringr, testthat, units, - webshot + webshot2 Suggests: kableExtra, - lwgeom, magick, oaStyle, rmarkdown diff --git a/reporting-grofwild/NAMESPACE b/reporting-grofwild/NAMESPACE index e4c94e7c..5a7a3168 100644 --- a/reporting-grofwild/NAMESPACE +++ b/reporting-grofwild/NAMESPACE @@ -173,6 +173,7 @@ importFrom(aws.s3,s3read_using) importFrom(aws.s3,s3save) importFrom(aws.s3,s3write_using) importFrom(aws.s3,save_object) +importFrom(aws.signature,read_credentials) importFrom(config,get) importFrom(data.table,rbindlist) importFrom(flexdashboard,gauge) @@ -238,4 +239,4 @@ importFrom(utils,read.csv) importFrom(utils,tail) importFrom(utils,write.csv) importFrom(utils,write.table) -importFrom(webshot,webshot) +importFrom(webshot2,webshot) diff --git a/reporting-grofwild/R/data_s3.R b/reporting-grofwild/R/data_s3.R index 7dce68b2..cafe2dbf 100644 --- a/reporting-grofwild/R/data_s3.R +++ b/reporting-grofwild/R/data_s3.R @@ -3,23 +3,38 @@ #' #' INFO: https://www.gormanalysis.com/blog/connecting-to-aws-s3-with-r/ #' @param awsFile path to AWS file +#' @param inboUserName set NULL if not a inbo user, for inbo user, provide your user name firstName_lastName #' @return no return value, ENV variables are set correctly #' #' @author mvarewyck +#' @importFrom aws.signature read_credentials #' @export -setupS3 <- function(awsFile = "~/.aws/credentials") { +setupS3 <- function(awsFile = "~/.aws/credentials", inboUserName = NULL) { + + #"sander_devisscher" + profile <- if (is.null(inboUserName)) + "inbo-fauna" else + sprintf("inbo-uat-%s", gsub("_", "-", inboUserName)) + + # for inbo user on their PC + userProfile <- Sys.getenv("USERPROFILE", unset = NA) + + + if (!is.na(userProfile)) + awsFile <- normalizePath(file.path(userProfile, ".aws", "credentials")) + # credentials are in ~/.aws/credentials OR manually copy/paste OR using aws.signature:: - x <- rawToChar(readBin(awsFile, "raw", n = 1e5L)) - profile <- Sys.getenv("AWS_PROFILE") - credentials <- strsplit(x, profile)[[1]][2] + x <- aws.signature::read_credentials(file = awsFile)[[profile]] + Sys.setenv( AWS_DEFAULT_REGION = eval(parse(text = config::get("credentials", file = system.file("config.yml", package = "reportingGrofwild"))$region)), - AWS_ACCESS_KEY_ID = strsplit(strsplit(credentials, "aws_access_key_id = ")[[1]][2], "\n")[[1]][1], - AWS_SECRET_ACCESS_KEY = strsplit(strsplit(credentials, "aws_secret_access_key = ")[[1]][2], "\n")[[1]][1] + AWS_ACCESS_KEY_ID = x$AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY = x$AWS_SECRET_ACCESS_KEY, + AWS_SESSION_TOKEN = x$AWS_SESSION_TOKEN ) - + } @@ -59,7 +74,7 @@ checkS3 <- function() { # Try to retrieve metadata from the instance metadata$instance_id() } else { - credentials <- Sys.getenv(c("AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")) + credentials <- Sys.getenv(c("AWS_DEFAULT_REGION", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", 'AWS_SESSION_TOKEN')) if (any(credentials == "")) stop("Please specify 'Sys.setenv()' for ", paste(names(credentials)[which(credentials == "")], collapse = ", ")) diff --git a/reporting-grofwild/R/mapFlanders.R b/reporting-grofwild/R/mapFlanders.R index 0bee4192..e44911d9 100644 --- a/reporting-grofwild/R/mapFlanders.R +++ b/reporting-grofwild/R/mapFlanders.R @@ -456,7 +456,7 @@ mapFlanders <- function( #' @import shiny #' @import leaflet #' @importFrom sf st_coordinates -#' @importFrom webshot webshot +#' @importFrom webshot2 webshot #' @importFrom htmlwidgets saveWidget #' @export mapFlandersServer <- function(id, defaultYear, species, currentWbe = reactive(NULL), @@ -1093,7 +1093,7 @@ mapFlandersServer <- function(id, defaultYear, species, currentWbe = reactive(NU htmlwidgets::saveWidget(finalMap(), file = tmpFile, selfcontained = FALSE) # convert temp .html file into .png for download - webshot::webshot(url = tmpFile, file = file, + webshot2::webshot(url = tmpFile, file = file, vwidth = 1000, vheight = 500, cliprect = "viewport") } diff --git a/reporting-grofwild/R/mapSchade.R b/reporting-grofwild/R/mapSchade.R index ebecc1b2..736dae3b 100644 --- a/reporting-grofwild/R/mapSchade.R +++ b/reporting-grofwild/R/mapSchade.R @@ -274,7 +274,7 @@ mapSchade <- function( #' #' @author mvarewyck #' @import shiny -#' @importFrom webshot webshot +#' @importFrom webshot2 webshot #' @importFrom leaflet renderLeaflet setView leafletProxy clearTiles #' @export mapSchadeServer <- function(id, schadeData, allSpatialData, timeRange, @@ -554,7 +554,7 @@ mapSchadeServer <- function(id, schadeData, allSpatialData, timeRange, htmlwidgets::saveWidget(results$perceelMap(), file = htmlFile, selfcontained = FALSE) # convert temp .html file into .png for download - webshot::webshot(url = htmlFile, file = pngFile, + webshot2::webshot(url = htmlFile, file = pngFile, vwidth = 1000, vheight = 500, cliprect = "viewport") # save in reactive value diff --git a/reporting-grofwild/R/mapSpread.R b/reporting-grofwild/R/mapSpread.R index d537c457..8ae0a150 100644 --- a/reporting-grofwild/R/mapSpread.R +++ b/reporting-grofwild/R/mapSpread.R @@ -181,7 +181,7 @@ mapVerkeer <- function(trafficData, layers = c("oversteek", "ecorasters"), #' @author mvarewyck #' @import shiny #' @import leaflet -#' @importFrom webshot webshot +#' @importFrom webshot2 webshot #' @importFrom htmlwidgets saveWidget #' @export mapSpreadServer <- function(id, regionLevel, locaties, allSpatialData, @@ -471,7 +471,7 @@ mapSpreadServer <- function(id, regionLevel, locaties, allSpatialData, htmlwidgets::saveWidget(finalMap(), file = tmpFile, selfcontained = FALSE) # convert temp .html file into .png for download - webshot::webshot(url = tmpFile, file = file, + webshot2::webshot(url = tmpFile, file = file, vwidth = 1000, vheight = 500, cliprect = "viewport") } diff --git a/reporting-grofwild/inst/NEWS b/reporting-grofwild/inst/NEWS index 5f6ef663..33253b4b 100644 --- a/reporting-grofwild/inst/NEWS +++ b/reporting-grofwild/inst/NEWS @@ -1,3 +1,7 @@ +0.3.6 + o replace webshot by webshot2 + o convert dockerfile for ARM64 + o mimick aws configuration alien species portal 0.3.5 o add onderkaaklengte gauge bioindicator #332 o add faunabeheerzones as choice for multiple graphs #473 diff --git a/reporting-grofwild/inst/config.yml b/reporting-grofwild/inst/config.yml index 5e602ac4..892a64a6 100644 --- a/reporting-grofwild/inst/config.yml +++ b/reporting-grofwild/inst/config.yml @@ -1,9 +1,10 @@ default: - bucket: inbo-wbe-uat-data + bucket: inbo-faunabeheer-uat-eu-west-1-default credentials: region: Sys.getenv('AWS_DEFAULT_REGION', unset = 'eu-west-1') id: Sys.getenv('AWS_ACCESS_KEY_ID') secret: Sys.getenv('AWS_SECRET_ACCESS_KEY') + token: Sys.getenv('AWS_SESSION_TOKEN') datacheck: false defaultYear: 2023 @@ -11,5 +12,5 @@ uat: datacheck: false production: - bucket: inbo-wbe-prd-data + bucket: inbo-faunabeheer-prd-eu-west-1-default datacheck: false diff --git a/reporting-grofwild/inst/ui/app_private/global.R b/reporting-grofwild/inst/ui/app_private/global.R index 7535cd24..1ecab245 100644 --- a/reporting-grofwild/inst/ui/app_private/global.R +++ b/reporting-grofwild/inst/ui/app_private/global.R @@ -22,6 +22,9 @@ addResourcePath("www", system.file("ui/www", package = "reportingGrofwild")) defaultYear <- as.numeric(format(Sys.Date(), "%Y")) - 1 +chromote::set_chrome_args(c('--headless','--no-sandbox')) + + ### Debugging ### ----------- diff --git a/reporting-grofwild/inst/ui/app_public/global.R b/reporting-grofwild/inst/ui/app_public/global.R index 186ea804..e9d1a5c6 100644 --- a/reporting-grofwild/inst/ui/app_public/global.R +++ b/reporting-grofwild/inst/ui/app_public/global.R @@ -18,6 +18,9 @@ dataDir <- system.file("extdata", package = "reportingGrofwild") addResourcePath("www", system.file("ui/www", package = "reportingGrofwild")) +chromote::set_chrome_args(c('--headless','--no-sandbox')) + + ### Debugging ### ----------- diff --git a/reporting-grofwild/inst/ui/www/plotDashboard.Rmd b/reporting-grofwild/inst/ui/www/plotDashboard.Rmd index 91ad4587..5e6fea94 100644 --- a/reporting-grofwild/inst/ui/www/plotDashboard.Rmd +++ b/reporting-grofwild/inst/ui/www/plotDashboard.Rmd @@ -75,7 +75,7 @@ tryCatch({ # leaflet tmpFile <- tempfile(fileext = ".html") htmlwidgets::saveWidget(params$plot, file = tmpFile, selfcontained = FALSE) - webshot::webshot(url = tmpFile, vwidth = 1000, vheight = 500, cliprect = "viewport") + webshot2::webshot(url = tmpFile, vwidth = 1000, vheight = 500, cliprect = "viewport") } else if (!is.null(params$pdfTable)){ diff --git a/reporting-grofwild/inst/ui/www/reportDashboard.Rmd b/reporting-grofwild/inst/ui/www/reportDashboard.Rmd index 3fda32fc..5ee69aa5 100644 --- a/reporting-grofwild/inst/ui/www/reportDashboard.Rmd +++ b/reporting-grofwild/inst/ui/www/reportDashboard.Rmd @@ -26,7 +26,6 @@ knitr::opts_chunk$set( fig.width = 10, fig.height = 8 ) -options(tinytex.tlmgr_update = FALSE) ``` ```{r locaties} @@ -47,7 +46,7 @@ incProgress(amount = 0.1, detail = "Achtergrondinformatie") tmpFile <- tempfile(fileext = ".html") htmlwidgets::saveWidget(dash_results$dash_finalMap()$plot, file = tmpFile, selfcontained = FALSE) -webshot::webshot(url = tmpFile, vwidth = 1000, vheight = 500, cliprect = "viewport") +webshot2::webshot(url = tmpFile, vwidth = 1000, vheight = 500, cliprect = "viewport") ``` ## Biotoop beschrijving diff --git a/template.Dockerfile b/template.Dockerfile deleted file mode 100644 index faf629b0..00000000 --- a/template.Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -#include packamon.disclaimer - -#include packamon.from - -#include packamon.system-dependencies - -#include packamon.r-repos - -#include packamon.r-dependencies - -# Missing installation - rmd pdf -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl -RUN R -e "webshot::install_phantomjs()" - -# For the rmarkdown pdf report -RUN R -e "tinytex::install_tinytex()" -ENV PATH="/root/bin:${PATH}" -RUN R -e "tinytex::tlmgr_install(pkgs = c('fancyhdr', 'sectsty', 'titling', 'grffile'))" - - -#include packamon.local-r-dependencies - -#include packamon.runtime-settings