Merge pull request #2 from TIBHannover/1-ingest-a-set-of-ontologies

implemented an ingestion pipeline for #1
TIBHannover · May 28, 2024 · 12c07a7 · 12c07a7
2 parents 9e6dab3 + 8d2ff95
commit 12c07a7
Show file tree

Hide file tree

Showing 5 changed files with 114 additions and 51 deletions.
diff --git a/ansible/playbook.yml b/ansible/playbook.yml
@@ -1,4 +1,5 @@
 
 - hosts: all
   roles:
+    - preprocess
     - ols
diff --git a/ansible/playbook_ingest.yml b/ansible/playbook_ingest.yml
@@ -0,0 +1,5 @@
+
+- hosts: all
+  roles:
+    - preprocess
+    - ingest
diff --git a/ansible/roles/ingest/tasks/main.yml b/ansible/roles/ingest/tasks/main.yml
@@ -0,0 +1,59 @@
+
+- name: mvn clean install
+  command: 'mvn clean install'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+  environment:
+        OLS4_CONFIG: '{{ ols4_config_file }}'
+
+- name: rdf2json
+  command: 'java -jar rdf2json/target/rdf2json-1.0-SNAPSHOT.jar --config {{ ols4_config_file }} --output {{ ols4_config_file.split('.')[0] }}_out.json'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: rm -rf output_csv && mkdir output_csv
+  command: 'rm -rf output_csv && mkdir output_csv'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: json2neo
+  command: 'java -jar json2neo/target/json2neo-1.0-SNAPSHOT.jar --input {{ ols4_config_file.split('.')[0] }}_out.json --outDir output_csv'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: json2solr
+  command: 'java -jar json2solr/target/json2solr-1.0-SNAPSHOT.jar  --input {{ ols4_config_file.split('.')[0] }}_out.json --outDir output_csv'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: wget ontologies
+  command: 'wget --method POST --no-proxy -O - --server-response --content-on-error=on --header="Content-Type: application/json" --body-file output_csv/ontologies.jsonl http://127.0.0.1:8983/solr/ols4_entities/update/json/docs?commit=true'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: wget classes
+  command: 'wget --method POST --no-proxy -O - --server-response --content-on-error=on --header="Content-Type: application/json" --body-file output_csv/classes.jsonl http://127.0.0.1:8983/solr/ols4_entities/update/json/docs?commit=true'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: wget properties
+  command: 'wget --method POST --no-proxy -O - --server-response --content-on-error=on --header="Content-Type: application/json" --body-file output_csv/properties.jsonl http://127.0.0.1:8983/solr/ols4_entities/update/json/docs?commit=true'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: wget individuals
+  command: 'wget --method POST --no-proxy -O - --server-response --content-on-error=on --header="Content-Type: application/json" --body-file output_csv/individuals.jsonl http://127.0.0.1:8983/solr/ols4_entities/update/json/docs?commit=true'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: wget autocomplete
+  command: 'wget --method POST --no-proxy -O - --server-response --content-on-error=on --header="Content-Type: application/json" --body-file output_csv/autocomplete.jsonl http://127.0.0.1:8983/solr/ols4_autocomplete/update/json/docs?commit=true'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+- name: csv2neo
+  command: 'java -jar csv2neo/target/csv2neo-1.0-SNAPSHOT.jar -i -d output_csv'
+  args:
+    chdir: '{{ ols4_base_dir }}/dataload'
+
+
diff --git a/ansible/roles/ols/tasks/main.yml b/ansible/roles/ols/tasks/main.yml
@@ -1,54 +1,3 @@
-
-- name: checkout OLS4 locally
-  git:
-    repo: '{{ ols4_repo }}'
-    dest: '{{ ols4_base_dir }}'
-    force: yes
-    version: '{{ ols4_version }}'
-
-- name: Copy dataload config files
-  copy:
-    src: '{{ item }}'
-    dest: '{{ ols4_base_dir }}/dataload/configs/'
-  when: (dataload_config_files is defined) and (dataload_config_files != "") and (dataload_config_files is not none)
-  with_items: '{{ dataload_config_files }}'
-
-- name: Copy dataload dockerfile
-  copy:
-    src: '{{ dataload_dockerfile }}'
-    dest: '{{ ols4_base_dir }}/dataload/'
-  when: (dataload_dockerfile is defined) and (dataload_dockerfile != "") and (dataload_dockerfile is not none)
-
-- name: Copy backend dockerfile
-  copy:
-    src: '{{ backend_dockerfile }}'
-    dest: '{{ ols4_base_dir }}/backend/'
-  when: (backend_dockerfile is defined) and (backend_dockerfile != "") and (backend_dockerfile is not none)
-
-- name: Copy frontend dockerfile
-  copy:
-    src: '{{ frontend_dockerfile }}'
-    dest: '{{ ols4_base_dir }}/frontend/'
-  when: (frontend_dockerfile is defined) and (frontend_dockerfile != "") and (frontend_dockerfile is not none)
-
-- name: Copy frontend envfile
-  copy:
-    src: '{{ frontend_envfile }}'
-    dest: '{{ ols4_base_dir }}/frontend/.env'
-  when: (frontend_envfile is defined) and (frontend_envfile != "") and (frontend_envfile is not none)
-
-- name: Copy docker compose file
-  copy:
-    src: '{{ alternative_dockercomposefile }}'
-    dest: '{{ ols4_base_dir }}/docker-compose.yml'
-  when: (alternative_dockercomposefile is defined) and (alternative_dockercomposefile != "") and (alternative_dockercomposefile is not none)
-
-- name: Replace backend url for swagger
-  command: 'sed -i "s,{{ ols4_default_swagger_url }},{{ ols4_swagger_url }},g" backend/src/main/java/uk/ac/ebi/spot/ols/config/SwaggerConfig.java'
-  args:
-    chdir: '{{ ols4_base_dir }}'
-  when: ols4_backend_url is not none
-
 - name: Run docker-compose up -d
   command: 'docker-compose up -d'
   args:

diff --git a/ansible/roles/preprocess/tasks/main.yml b/ansible/roles/preprocess/tasks/main.yml
@@ -0,0 +1,49 @@
+- name: checkout OLS4 locally
+  git:
+    repo: '{{ ols4_repo }}'
+    dest: '{{ ols4_base_dir }}'
+    force: yes
+    version: '{{ ols4_version }}'
+
+- name: Copy dataload config files
+  copy:
+    src: '{{ item }}'
+    dest: '{{ ols4_base_dir }}/dataload/configs/'
+  when: (dataload_config_files is defined) and (dataload_config_files != "") and (dataload_config_files is not none)
+  with_items: '{{ dataload_config_files }}'
+
+- name: Copy dataload dockerfile
+  copy:
+    src: '{{ dataload_dockerfile }}'
+    dest: '{{ ols4_base_dir }}/dataload/'
+  when: (dataload_dockerfile is defined) and (dataload_dockerfile != "") and (dataload_dockerfile is not none)
+
+- name: Copy backend dockerfile
+  copy:
+    src: '{{ backend_dockerfile }}'
+    dest: '{{ ols4_base_dir }}/backend/'
+  when: (backend_dockerfile is defined) and (backend_dockerfile != "") and (backend_dockerfile is not none)
+
+- name: Copy frontend dockerfile
+  copy:
+    src: '{{ frontend_dockerfile }}'
+    dest: '{{ ols4_base_dir }}/frontend/'
+  when: (frontend_dockerfile is defined) and (frontend_dockerfile != "") and (frontend_dockerfile is not none)
+
+- name: Copy frontend envfile
+  copy:
+    src: '{{ frontend_envfile }}'
+    dest: '{{ ols4_base_dir }}/frontend/.env'
+  when: (frontend_envfile is defined) and (frontend_envfile != "") and (frontend_envfile is not none)
+
+- name: Copy docker compose file
+  copy:
+    src: '{{ alternative_dockercomposefile }}'
+    dest: '{{ ols4_base_dir }}/docker-compose.yml'
+  when: (alternative_dockercomposefile is defined) and (alternative_dockercomposefile != "") and (alternative_dockercomposefile is not none)
+
+- name: Replace backend url for swagger
+  command: 'sed -i "s,{{ ols4_default_swagger_url }},{{ ols4_swagger_url }},g" backend/src/main/java/uk/ac/ebi/spot/ols/config/SwaggerConfig.java'
+  args:
+    chdir: '{{ ols4_base_dir }}'
+  when: ols4_backend_url is not none