Merge branch 'master' into chore/fix-google-replication

uc-cdis · Jan 14, 2025 · 278f669 · 278f669
2 parents 72e44a7 + fef3d7c
commit 278f669
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 9 deletions.
diff --git a/gen3/bin/kube-setup-gen3-user-data-library.sh b/gen3/bin/kube-setup-gen3-user-data-library.sh
@@ -13,10 +13,6 @@ setup_database() {
     gen3_log_info "gen3userdatalibrary-g3auto secret already configured"
     return 0
   fi
-  if [[ -n "$JENKINS_HOME" || ! -f "$(gen3_secrets_folder)/creds.json" ]]; then
-    gen3_log_err "skipping db setup in non-adminvm environment"
-    return 0
-  fi
   # Setup .env file that gen3-user-data-library service consumes
   if [[ ! -f "$secretsFolder/gen3-user-data-library.env" || ! -f "$secretsFolder/base64Authz.txt" ]]; then
     local secretsFolder="$(gen3_secrets_folder)/g3auto/gen3userdatalibrary"

diff --git a/kube/services/jobs/indexd-single-table-migration-job.yaml b/kube/services/jobs/indexd-single-table-migration-job.yaml
@@ -1,4 +1,5 @@
 # Setup for running this migration https://github.com/uc-cdis/indexd/blob/master/docs/migration_to_single_table_indexd.md
+# Need to apply PriorityClass before running this job. Run kubectl apply -f indexd-single-table-migration-priority-class.yaml
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -9,6 +10,7 @@ spec:
       labels:
         app: gen3job
     spec:
+      priorityClassName: indexd-single-table-migration-priority  # Assigning the priority class
       affinity:
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -46,6 +48,12 @@ spec:
           env:
             - name: START_DID
               GEN3_START_DID|-value: ""-|
+            - name: BATCH_SIZE
+              GEN3_BATCH_SIZE|-value: ""-|
+            - name: OFFSET_VALUE
+              GEN3_OFFSET_VALUE|-value: ""-|
+            - name: USE_DEFAULT_CREDS
+              GEN3_USE_DEFAULT_CREDS|-value: "false"-|
           volumeMounts:
             - name: "config-volume"
               readOnly: true
@@ -72,11 +80,28 @@ spec:
           args:
             - "-c"
             - |
-              flags="--creds-path /var/www/indexd/creds_new.json"
+              $flags=""
+              
+              if [[ "$USE_DEFAULT_CREDS" == "false" ]]; then
+                echo "Using new creds"
+                flags="$flags --creds-file /var/www/indexd/creds_new.json"
+              else
+                echo "using default creds"
+                flags="$flags --creds-file /var/www/indexd/creds.json"
+              fi
+
               if [[ -n "$START_DID" ]]; then
                 flags="$flags --start-did $START_DID"
               fi
-              time poetry run python /indexd/bin/migrate_to_single_table.py $flags
+              if [[ -n "$BATCH_SIZE" ]]; then
+                flags="$flags --batch-size $BATCH_SIZE"
+              fi
+
+              if [[ -n "$OFFSET_VALUE" ]]; then
+                flags="$flags --start-offset $OFFSET_VALUE"
+              fi
+
+              time python /indexd/bin/migrate_to_single_table.py $flags
               echo "Exit code: $?"
       restartPolicy: Never
 

diff --git a/kube/services/jobs/indexd-single-table-migration-priority-class.yaml b/kube/services/jobs/indexd-single-table-migration-priority-class.yaml
@@ -0,0 +1,7 @@
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: indexd-single-table-migration-priority
+value: 100000  # High value for high priority
+globalDefault: false
+description: "Priority class for Indexd migration job"
diff --git a/kube/services/spark/spark-deploy.yaml b/kube/services/spark/spark-deploy.yaml
@@ -87,9 +87,6 @@ spec:
           args: 
             - "-c"
             - |
-              # get /usr/local/share/ca-certificates/cdis-ca.crt into system bundle
-              ssh server sudo /etc/init.d/ssh start
-              update-ca-certificates
               python run_config.py
               hdfs namenode -format
               hdfs --daemon start namenode