Skip to content

Commit

Permalink
Workflow dispatch templates (#174)
Browse files Browse the repository at this point in the history 
* delete old templates code, try deploy docker im devel

* update r-im creator branch

* get DESCRIPTION from admiral

* comment other deploys

* change credentials with pharmaverse-bot

* update push devel

* add artifact

* fix description URI

* fix creds

* FIX

* fix image name

* updates

* clean

---------

Co-authored-by: pharmaverse-bot <[email protected]>
  • Loading branch information
@dgrassellyb @pharmaverse-bot
dgrassellyb and pharmaverse-bot authored Dec 4, 2023
1 parent 1345e31 commit f52550a
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 309 deletions.
3 changes: 2 additions & 1 deletion .Rbuildignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,5 @@
^admiralci.*\.tgz$
^staged_dependencies.yaml$
^README.Rmd$
^.devcontainer
^.devcontainer
^install_devel_deps.R
294 changes: 0 additions & 294 deletions .github/workflows/check-templates.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,56 +10,14 @@ on:
- '4.1'
- '4.2'
- '4.3'
push-templates-data:
description: 'Push generated templates data to pharmaverseadam repo'
default: false
required: false
type: boolean
exclude-templates-data:
description: 'List of data generated by templates to exclude (templates script will run, but results will not be push to pharmaverseadam repo) - comma seperated list'
default: ''
required: false
type: string
workflow_call:
inputs:
r-version:
description: 'The version of R to use'
default: '4.1'
required: false
type: string
push-templates-data:
description: 'Push generated templates data to pharmaverseadam repo'
default: false
required: false
type: boolean
exclude-templates-data:
description: 'List of data generated by templates to exclude (templates script will run, but results will not be push to pharmaverseadam repo) - comma seperated list'
default: ''
required: false
type: string

env:
# branch name for PR
source-branch: update_templates_data
repo: pharmaverseadam
target-branch: main
reviewers: >
{
"admiralonco": {
"reviewers": ["bundfussr", "manciniedoardo"]
},
"admiralophtha": {
"reviewers": ["manciniedoardo"]
},
"admiral": {
"reviewers": ["manciniedoardo", "bms63"]
},
"admiralvaccine": {
"reviewers": ["ahasoplakus", "arjoon-r", "manciniedoardo"]
}
}
# data folder name for templates
templates_data_folder: ./data/

name: Check Templates

Expand Down Expand Up @@ -185,255 +143,3 @@ jobs:
}
}
shell: Rscript {0}

- name: Add other data formats and suffixes, exclude data
if: inputs.push-templates-data
run: |
library(readxl)
library(zip)
loadRData <- function(fileName){
load(fileName)
get(ls()[ls() != "fileName"])
}
print(file.path("${{ github.workspace }}", "tmp"))
print(file.path(Sys.getenv("GITHUB_WORKSPACE"), "tmp"))
folder_path <- file.path(Sys.getenv("GITHUB_WORKSPACE"), "tmp")
rda_files <- list.files(path = folder_path, pattern = "\\.rda$")
ex_files <- unlist(strsplit("${{ inputs.exclude-templates-data }}", ",")) # files to exclude
for (rda_file in rda_files) {
# delete file if part of inputs.exclude-templates-data
if (gsub("\\.rda$", "", rda_file) %in% ex_files) {
file.remove(file.path(folder_path, rda_file))
cat("Deleted:", rda_file, "\n")
}
else {
print(sprintf("converting file %s", rda_file))
data <- loadRData(file.path(folder_path, rda_file))
suffix <- gsub("pharmaverse/admiral", "", "${{ github.repository }}")
if (nchar(suffix) > 0) {
rda_file_renamed <- gsub("\\.rda$", sprintf("_%s.rda", suffix), rda_file)
csv_file <- gsub("\\.rda$", sprintf("_%s.csv", suffix), rda_file)
file.rename(file.path(folder_path, rda_file), file.path(folder_path, rda_file_renamed))
}
else {
csv_file <- gsub("\\.rda$", ".csv", rda_file)
rda_file_renamed <- rda_file
}
write.csv(data, file = file.path(folder_path, csv_file), row.names = FALSE)
# rename content of rda file
rda_var <- gsub("\\.rda$", "", rda_file_renamed)
print(rda_var)
assign(rda_var, data)
do.call(save, list(rda_var, file = file.path(folder_path, sprintf("%s.rda", rda_var)), compress = "bzip2"))
# create associated documentation inside R folder
dataset_name <- gsub("\\.rda$", "", rda_file_renamed)
doc_string <- paste(
sprintf("#' Dataset %s", dataset_name),
"#'",
sprintf("#' %s dataset", dataset_name),
"#'",
sprintf("#' @name %s", dataset_name),
"#' @docType data",
sprintf("#' @format A data frame with %s columns:", ncol(data)),
"#' \\describe{",
paste(sapply(names(data), function(col_name) {
paste(sprintf("#' \\item{ %s }{%s}", col_name, col_name))
}, USE.NAMES = FALSE), collapse="\n"),
"#' }",
"#'", sprintf("#' @source Generated from ${{ github.repository }}."),
"#' @references None",
"#'", sprintf("#' @examples\n#' data(\"%s\")", dataset_name),
sep = '\n',
sprintf("\"%s\"", dataset_name)
)
doc_dir <- "datasets_doc"
if (!file.exists(doc_dir)) {
dir.create(doc_dir, recursive = TRUE)
}
writeLines(doc_string, con = file.path(doc_dir, paste0(dataset_name, ".R")))
}
}
shell: Rscript {0}

# zip templates data
- name: zip artifacts data
if: inputs.push-templates-data
run: |
find "$GITHUB_WORKSPACE/tmp" -type f \( -name "*.rda" -o -name "*.csv" \) -exec zip -j "$GITHUB_WORKSPACE/data.zip" {} \;
find "$GITHUB_WORKSPACE/datasets_doc" -type f \( -name "*.R" \) -exec zip -j "$GITHUB_WORKSPACE/doc.zip" {} \;
# store templates data as artifacts
- name: Archive templates data
if: inputs.push-templates-data
uses: actions/upload-artifact@v3
with:
name: data_templates
path: ${{ github.workspace }}/data.zip

# store templates data as artifacts
- name: Archive doc
if: inputs.push-templates-data
uses: actions/upload-artifact@v3
with:
name: doc_templates
path: ${{ github.workspace }}/doc.zip

- name: Checkout repo (PR) 🛎
if: inputs.push-templates-data
uses: actions/checkout@v3
with:
ref: main
repository: "${{ github.repository_owner }}/${{ env.repo }}"
fetch-depth: 2
token: ${{ secrets.PHARMAVERSE_BOT }}

- name: Add safe directory
if: inputs.push-templates-data
run: |
git config --global --add safe.directory "${GITHUB_WORKSPACE}"
shell: bash

- name: Download data artifacts
if: inputs.push-templates-data
uses: actions/download-artifact@v3
with:
name: data_templates

- name: Download doc artifacts
if: inputs.push-templates-data
uses: actions/download-artifact@v3
with:
name: doc_templates

# detect diffs om templates data
- name: Check templates changes
if: inputs.push-templates-data
id: changes
run: |
head_data_folder=$(git ls-tree -r HEAD data) # check if data folder exists on remote main branch
unzip -o "${GITHUB_WORKSPACE}/data.zip" -d "${GITHUB_WORKSPACE}/inst/extdata/"
unzip -o "${GITHUB_WORKSPACE}/doc.zip" -d "${GITHUB_WORKSPACE}/R"
# move every rda files inside data folder
mkdir -p data
for rda_file in inst/extdata/*.rda; do
filename=$(basename "$rda_file")
mv "$rda_file" "data/$filename"
done
if [ -z "${head_data_folder}" ]
then {
echo "data folder not existing on main branch yet - init mode (all templates data files will be pushed)"
diff_files=$(ls "${GITHUB_WORKSPACE}/data")
} else {
git add data R
diff_files_data=$(git status data | grep 'data/.*\.rda$')
diff_files_doc=$(git status R | grep 'R/.*\.R$')
diff_files="${diff_files_data}${diff_files_doc}"
}
fi
if [ -z "$diff_files" ]
then {
echo "No changes detected after running templates. Stop here"
echo "diff=false" >> $GITHUB_OUTPUT
} else {
echo "Changes detected, new PR will be created"
echo "list of files with new updates: $diff_files"
echo "diff=true" >> $GITHUB_OUTPUT
}
fi
shell: bash {0}

- name: Generate man pages
if: inputs.push-templates-data
run: |
Rscript -e "roxygen2::roxygenize('.', roclets = c('rd', 'collate', 'namespace'))"
git add man
shell: bash {0}

- name: Update spellchecks list
if: inputs.push-templates-data
run: |
spelling::update_wordlist(pkg=".", confirm=FALSE)
shell: Rscript {0}

- name: set source-branch name
if: inputs.push-templates-data
id: branch
run: |
repo_name=$(basename "${{ github.repository }}")
source_br="${{ env.source-branch }}_${repo_name}"
source_grep=$(git branch -a | grep $source_br)
if [ -z $source_grep ]; then branch_exists=false; else branch_exists=true; fi
echo "source-branch=$source_br" >> $GITHUB_OUTPUT
echo "branch-exists=$branch_exists" >> $GITHUB_OUTPUT
warn_msg="WARNING:: the branch $source_br already exists in pharmaverseadam, please merge it and delete it, and re-execute this job"
if [ '${{ steps.changes.outputs.diff }}' == 'true' ] && [ '$branch_exists' == 'true' ]; then echo "$warn_msg"; fi
shell: bash {0}

# if diff detected, push changes
- name: Commit and push changes in {{ env.repo }}
if: ${{ inputs.push-templates-data && steps.changes.outputs.diff == 'true' && steps.branch.outputs.branch-exists == 'false' }}
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: templates data updates from ${{ github.repository }}
file_pattern: 'inst/extdata/* data/* R/* inst/WORDLIST'
commit_user_name: pharmaverse-bot
commit_user_email: [email protected]
branch: "${{ steps.branch.outputs.source-branch }}"
create_branch: true

- name: Create Pull Request
if: ${{ inputs.push-templates-data && steps.changes.outputs.diff == 'true' && steps.branch.outputs.branch-exists == 'false' }}
uses: actions/github-script@v6
with:
github-token: ${{ secrets.PHARMAVERSE_BOT }}
script: |
// Look for any open PRs
const repo_owner = "${{ github.repository_owner }}";
const target_repo_name = "${{ env.repo }}";
const result = await github.rest.pulls.list({
owner: repo_owner,
repo: target_repo_name,
state: "open"
})
let create_new_pr = true;
for (const pr of result.data) {
// Look for distinct PR branch name
if (pr.head.ref === "${{ steps.branch.outputs.source-branch }}") {
console.log("PR with head ref " + pr.head.ref + " already exists");
create_new_pr = false;
break;
}
}
// If no PR with distinguished branch name has been found
// create a new PR to track changes to renv.lock.
if (create_new_pr) {
console.log("Creating a new PR");
const result2 = await github.rest.pulls.create({
title: 'Templates data updates from ${{ github.repository }}',
owner: repo_owner,
repo: target_repo_name,
head: '${{ steps.branch.outputs.source-branch }}',
base: '${{ env.target-branch }}',
body: [
'This PR has been automatically generated by ',
'check-templates workflow from ${{ github.repository }}.',
'\n\nPlease review the changes.'
].join('')
});
// Assign reviewers to the PR
let rev = process.env.reviewers;
let reviewers_dict = JSON.parse(rev);
let current_repo_name_full = "${{ github.repository }}";
let current_repo_name = current_repo_name_full.split('/').pop();
const result3 = await github.rest.pulls.requestReviewers({
owner: repo_owner,
repo: target_repo_name,
pull_number: result2.data.number,
reviewers: reviewers_dict[current_repo_name]["reviewers"]
});
}
39 changes: 25 additions & 14 deletions .github/workflows/push-docker-image.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,29 +105,40 @@ jobs:
packages: write

steps:
- name: Checkout repo
uses: actions/checkout@v3
with:
fetch-depth: 0

- name: Set image specs
id: image_specs
run: |
package_name=$(grep "Package:" DESCRIPTION | awk '{print $NF}')
echo "image_name=$package_name-devel" >> $GITHUB_OUTPUT

- name: Call deploy docker image action devel
uses: insightsengineering/r-image-creator@v1
with:
image-name: "${{ steps.image_specs.outputs.image_name }}"
image-name: "admiralci-devel"
tag-latest: true
base-image: "rocker/rstudio:devel"
sysdeps: qpdf,libxt-dev,curl,npm,libicu-dev,libcurl4-openssl-dev,libssl-dev,make,zlib1g-dev,libfontconfig1-dev,libfreetype6-dev,libfribidi-dev,libharfbuzz-dev,libjpeg-dev,libpng-dev,libtiff-dev,pandoc,libxml2-dev,libgit2-dev,libgit2-dev,jq
description-file: "./DESCRIPTION"
description-file: "https://raw.githubusercontent.com/pharmaverse/admiral/main/DESCRIPTION"
repository-owner: ${{ github.repository_owner }}
repo-user: ${{ github.actor }} # pharmaverse-bot
repo-token: "${{ secrets.GITHUB_TOKEN }}" # ${{ secrets.PHARMAVERSE_BOT }}
repo-user: ${{ github.actor }}
repo-token: "${{ secrets.GITHUB_TOKEN }}"

store_deps:
name: Store Deps
runs-on: ubuntu-latest
needs: [ deploy-image-devel ]
container:
image: "ghcr.io/pharmaverse/admiralci-devel:latest"
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
steps:
##################### BEGIN boilerplate steps #####################
- name: Store deps into csv file
id: branch-name
run: |
installed_packages <- as.data.frame(installed.packages())
write.csv(installed_packages, "/workspace/deps.csv", row.names = FALSE)
shell: Rscript {0}
- name: Upload deps.csv artifact
uses: actions/upload-artifact@v2
with:
name: deps
path: "/workspace/deps.csv"

# refacto todo: image-name, sysdeps on get-renv-list job

Expand Down
9 changes: 9 additions & 0 deletions install_devel_deps.R
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Read the dependencies from the CSV file
dependencies <- read.csv("deps.csv", stringsAsFactors = FALSE)

# Loop through the dependencies and reinstall packages with specific versions
for (i in seq_len(nrow(dependencies))) {
dep_name <- dependencies[i, "Package"]
dep_version <- dependencies[i, "Version"]
install.packages(dep_name, type = "source", repos = NULL, version = dep_version)
}

0 comments on commit f52550a

Please sign in to comment.