From 7f9e701918ad70ebcc8de63bb9799cf77a3e38c4 Mon Sep 17 00:00:00 2001 From: jaserud Date: Tue, 24 Aug 2021 10:58:28 -0400 Subject: [PATCH 01/26] next snapshot --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5fa2f05..8c97f80 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.icgc.argo workflow-management - 3.1.0 + 3.2.0-SNAPSHOT workflow-management ARGO Workflow Management From 879d6aa2f2b37ddcf98422a4b30c2d52dc797cb6 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 23 Aug 2021 15:25:38 -0400 Subject: [PATCH 02/26] new util to extract volMount from workDir given config --- .../streams/schema/EngineParams.java | 983 +++++++++++++++ .../streams/schema/RunState.java | 32 + .../streams/schema/WfMgmtRunMsg.java | 1115 +++++++++++++++++ .../util/VolumeMounts.java | 30 + .../wes/NextflowService.java | 11 +- .../wes/properties/NextflowProperties.java | 4 +- src/main/resources/application.yml | 2 + .../workflow_management/VolumeMountsTest.java | 68 + 8 files changed, 2240 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java create mode 100644 src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java create mode 100644 src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java create mode 100644 src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java create mode 100644 src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java diff --git a/src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java b/src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java new file mode 100644 index 0000000..7e33b46 --- /dev/null +++ b/src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java @@ -0,0 +1,983 @@ +/** + * Autogenerated by Avro + * + *

DO NOT EDIT DIRECTLY + */ +package org.icgc.argo.workflow_management.streams.schema; + +import org.apache.avro.message.BinaryMessageDecoder; +import org.apache.avro.message.BinaryMessageEncoder; +import org.apache.avro.message.SchemaStore; +import org.apache.avro.specific.SpecificData; + +@org.apache.avro.specific.AvroGenerated +public class EngineParams extends org.apache.avro.specific.SpecificRecordBase + implements org.apache.avro.specific.SpecificRecord { + private static final long serialVersionUID = 4292292046586660533L; + public static final org.apache.avro.Schema SCHEMA$ = + new org.apache.avro.Schema.Parser() + .parse( + "{\"type\":\"record\",\"name\":\"EngineParams\",\"namespace\":\"org.icgc.argo.workflow_management.streams.schema\",\"fields\":[{\"name\":\"defaultContainer\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"revision\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"launchDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"projectDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"latest\",\"type\":[\"null\",\"boolean\"],\"default\":null},{\"name\":\"resume\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null}]}"); + + public static org.apache.avro.Schema getClassSchema() { + return SCHEMA$; + } + + private static SpecificData MODEL$ = new SpecificData(); + + private static final BinaryMessageEncoder ENCODER = + new BinaryMessageEncoder(MODEL$, SCHEMA$); + + private static final BinaryMessageDecoder DECODER = + new BinaryMessageDecoder(MODEL$, SCHEMA$); + + /** + * Return the BinaryMessageEncoder instance used by this class. + * + * @return the message encoder used by this class + */ + public static BinaryMessageEncoder getEncoder() { + return ENCODER; + } + + /** + * Return the BinaryMessageDecoder instance used by this class. + * + * @return the message decoder used by this class + */ + public static BinaryMessageDecoder getDecoder() { + return DECODER; + } + + /** + * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link + * SchemaStore}. + * + * @param resolver a {@link SchemaStore} used to find schemas by fingerprint + * @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore + */ + public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { + return new BinaryMessageDecoder(MODEL$, SCHEMA$, resolver); + } + + /** + * Serializes this EngineParams to a ByteBuffer. + * + * @return a buffer holding the serialized data for this instance + * @throws java.io.IOException if this instance could not be serialized + */ + public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { + return ENCODER.encode(this); + } + + /** + * Deserializes a EngineParams from a ByteBuffer. + * + * @param b a byte buffer holding serialized data for an instance of this class + * @return a EngineParams instance decoded from the given buffer + * @throws java.io.IOException if the given bytes could not be deserialized into an instance of + * this class + */ + public static EngineParams fromByteBuffer(java.nio.ByteBuffer b) throws java.io.IOException { + return DECODER.decode(b); + } + + private java.lang.String defaultContainer; + private java.lang.String revision; + private java.lang.String launchDir; + private java.lang.String projectDir; + private java.lang.String workDir; + private java.lang.Boolean latest; + private java.lang.String resume; + + /** + * Default constructor. Note that this does not initialize fields to their default values from the + * schema. If that is desired then one should use newBuilder(). + */ + public EngineParams() {} + + /** + * All-args constructor. + * + * @param defaultContainer The new value for defaultContainer + * @param revision The new value for revision + * @param launchDir The new value for launchDir + * @param projectDir The new value for projectDir + * @param workDir The new value for workDir + * @param latest The new value for latest + * @param resume The new value for resume + */ + public EngineParams( + java.lang.String defaultContainer, + java.lang.String revision, + java.lang.String launchDir, + java.lang.String projectDir, + java.lang.String workDir, + java.lang.Boolean latest, + java.lang.String resume) { + this.defaultContainer = defaultContainer; + this.revision = revision; + this.launchDir = launchDir; + this.projectDir = projectDir; + this.workDir = workDir; + this.latest = latest; + this.resume = resume; + } + + public org.apache.avro.specific.SpecificData getSpecificData() { + return MODEL$; + } + + public org.apache.avro.Schema getSchema() { + return SCHEMA$; + } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: + return defaultContainer; + case 1: + return revision; + case 2: + return launchDir; + case 3: + return projectDir; + case 4: + return workDir; + case 5: + return latest; + case 6: + return resume; + default: + throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value = "unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: + defaultContainer = value$ != null ? value$.toString() : null; + break; + case 1: + revision = value$ != null ? value$.toString() : null; + break; + case 2: + launchDir = value$ != null ? value$.toString() : null; + break; + case 3: + projectDir = value$ != null ? value$.toString() : null; + break; + case 4: + workDir = value$ != null ? value$.toString() : null; + break; + case 5: + latest = (java.lang.Boolean) value$; + break; + case 6: + resume = value$ != null ? value$.toString() : null; + break; + default: + throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + /** + * Gets the value of the 'defaultContainer' field. + * + * @return The value of the 'defaultContainer' field. + */ + public java.lang.String getDefaultContainer() { + return defaultContainer; + } + + /** + * Sets the value of the 'defaultContainer' field. + * + * @param value the value to set. + */ + public void setDefaultContainer(java.lang.String value) { + this.defaultContainer = value; + } + + /** + * Gets the value of the 'revision' field. + * + * @return The value of the 'revision' field. + */ + public java.lang.String getRevision() { + return revision; + } + + /** + * Sets the value of the 'revision' field. + * + * @param value the value to set. + */ + public void setRevision(java.lang.String value) { + this.revision = value; + } + + /** + * Gets the value of the 'launchDir' field. + * + * @return The value of the 'launchDir' field. + */ + public java.lang.String getLaunchDir() { + return launchDir; + } + + /** + * Sets the value of the 'launchDir' field. + * + * @param value the value to set. + */ + public void setLaunchDir(java.lang.String value) { + this.launchDir = value; + } + + /** + * Gets the value of the 'projectDir' field. + * + * @return The value of the 'projectDir' field. + */ + public java.lang.String getProjectDir() { + return projectDir; + } + + /** + * Sets the value of the 'projectDir' field. + * + * @param value the value to set. + */ + public void setProjectDir(java.lang.String value) { + this.projectDir = value; + } + + /** + * Gets the value of the 'workDir' field. + * + * @return The value of the 'workDir' field. + */ + public java.lang.String getWorkDir() { + return workDir; + } + + /** + * Sets the value of the 'workDir' field. + * + * @param value the value to set. + */ + public void setWorkDir(java.lang.String value) { + this.workDir = value; + } + + /** + * Gets the value of the 'latest' field. + * + * @return The value of the 'latest' field. + */ + public java.lang.Boolean getLatest() { + return latest; + } + + /** + * Sets the value of the 'latest' field. + * + * @param value the value to set. + */ + public void setLatest(java.lang.Boolean value) { + this.latest = value; + } + + /** + * Gets the value of the 'resume' field. + * + * @return The value of the 'resume' field. + */ + public java.lang.String getResume() { + return resume; + } + + /** + * Sets the value of the 'resume' field. + * + * @param value the value to set. + */ + public void setResume(java.lang.String value) { + this.resume = value; + } + + /** + * Creates a new EngineParams RecordBuilder. + * + * @return A new EngineParams RecordBuilder + */ + public static org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder newBuilder() { + return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(); + } + + /** + * Creates a new EngineParams RecordBuilder by copying an existing Builder. + * + * @param other The existing builder to copy. + * @return A new EngineParams RecordBuilder + */ + public static org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder newBuilder( + org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder other) { + if (other == null) { + return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(); + } else { + return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(other); + } + } + + /** + * Creates a new EngineParams RecordBuilder by copying an existing EngineParams instance. + * + * @param other The existing instance to copy. + * @return A new EngineParams RecordBuilder + */ + public static org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder newBuilder( + org.icgc.argo.workflow_management.streams.schema.EngineParams other) { + if (other == null) { + return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(); + } else { + return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(other); + } + } + + /** RecordBuilder for EngineParams instances. */ + @org.apache.avro.specific.AvroGenerated + public static class Builder + extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private java.lang.String defaultContainer; + private java.lang.String revision; + private java.lang.String launchDir; + private java.lang.String projectDir; + private java.lang.String workDir; + private java.lang.Boolean latest; + private java.lang.String resume; + + /** Creates a new Builder */ + private Builder() { + super(SCHEMA$); + } + + /** + * Creates a Builder by copying an existing Builder. + * + * @param other The existing Builder to copy. + */ + private Builder(org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder other) { + super(other); + if (isValidValue(fields()[0], other.defaultContainer)) { + this.defaultContainer = data().deepCopy(fields()[0].schema(), other.defaultContainer); + fieldSetFlags()[0] = other.fieldSetFlags()[0]; + } + if (isValidValue(fields()[1], other.revision)) { + this.revision = data().deepCopy(fields()[1].schema(), other.revision); + fieldSetFlags()[1] = other.fieldSetFlags()[1]; + } + if (isValidValue(fields()[2], other.launchDir)) { + this.launchDir = data().deepCopy(fields()[2].schema(), other.launchDir); + fieldSetFlags()[2] = other.fieldSetFlags()[2]; + } + if (isValidValue(fields()[3], other.projectDir)) { + this.projectDir = data().deepCopy(fields()[3].schema(), other.projectDir); + fieldSetFlags()[3] = other.fieldSetFlags()[3]; + } + if (isValidValue(fields()[4], other.workDir)) { + this.workDir = data().deepCopy(fields()[4].schema(), other.workDir); + fieldSetFlags()[4] = other.fieldSetFlags()[4]; + } + if (isValidValue(fields()[5], other.latest)) { + this.latest = data().deepCopy(fields()[5].schema(), other.latest); + fieldSetFlags()[5] = other.fieldSetFlags()[5]; + } + if (isValidValue(fields()[6], other.resume)) { + this.resume = data().deepCopy(fields()[6].schema(), other.resume); + fieldSetFlags()[6] = other.fieldSetFlags()[6]; + } + } + + /** + * Creates a Builder by copying an existing EngineParams instance + * + * @param other The existing instance to copy. + */ + private Builder(org.icgc.argo.workflow_management.streams.schema.EngineParams other) { + super(SCHEMA$); + if (isValidValue(fields()[0], other.defaultContainer)) { + this.defaultContainer = data().deepCopy(fields()[0].schema(), other.defaultContainer); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.revision)) { + this.revision = data().deepCopy(fields()[1].schema(), other.revision); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.launchDir)) { + this.launchDir = data().deepCopy(fields()[2].schema(), other.launchDir); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.projectDir)) { + this.projectDir = data().deepCopy(fields()[3].schema(), other.projectDir); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.workDir)) { + this.workDir = data().deepCopy(fields()[4].schema(), other.workDir); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.latest)) { + this.latest = data().deepCopy(fields()[5].schema(), other.latest); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.resume)) { + this.resume = data().deepCopy(fields()[6].schema(), other.resume); + fieldSetFlags()[6] = true; + } + } + + /** + * Gets the value of the 'defaultContainer' field. + * + * @return The value. + */ + public java.lang.String getDefaultContainer() { + return defaultContainer; + } + + /** + * Sets the value of the 'defaultContainer' field. + * + * @param value The value of 'defaultContainer'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder + setDefaultContainer(java.lang.String value) { + validate(fields()[0], value); + this.defaultContainer = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'defaultContainer' field has been set. + * + * @return True if the 'defaultContainer' field has been set, false otherwise. + */ + public boolean hasDefaultContainer() { + return fieldSetFlags()[0]; + } + + /** + * Clears the value of the 'defaultContainer' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder + clearDefaultContainer() { + defaultContainer = null; + fieldSetFlags()[0] = false; + return this; + } + + /** + * Gets the value of the 'revision' field. + * + * @return The value. + */ + public java.lang.String getRevision() { + return revision; + } + + /** + * Sets the value of the 'revision' field. + * + * @param value The value of 'revision'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setRevision( + java.lang.String value) { + validate(fields()[1], value); + this.revision = value; + fieldSetFlags()[1] = true; + return this; + } + + /** + * Checks whether the 'revision' field has been set. + * + * @return True if the 'revision' field has been set, false otherwise. + */ + public boolean hasRevision() { + return fieldSetFlags()[1]; + } + + /** + * Clears the value of the 'revision' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearRevision() { + revision = null; + fieldSetFlags()[1] = false; + return this; + } + + /** + * Gets the value of the 'launchDir' field. + * + * @return The value. + */ + public java.lang.String getLaunchDir() { + return launchDir; + } + + /** + * Sets the value of the 'launchDir' field. + * + * @param value The value of 'launchDir'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setLaunchDir( + java.lang.String value) { + validate(fields()[2], value); + this.launchDir = value; + fieldSetFlags()[2] = true; + return this; + } + + /** + * Checks whether the 'launchDir' field has been set. + * + * @return True if the 'launchDir' field has been set, false otherwise. + */ + public boolean hasLaunchDir() { + return fieldSetFlags()[2]; + } + + /** + * Clears the value of the 'launchDir' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearLaunchDir() { + launchDir = null; + fieldSetFlags()[2] = false; + return this; + } + + /** + * Gets the value of the 'projectDir' field. + * + * @return The value. + */ + public java.lang.String getProjectDir() { + return projectDir; + } + + /** + * Sets the value of the 'projectDir' field. + * + * @param value The value of 'projectDir'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setProjectDir( + java.lang.String value) { + validate(fields()[3], value); + this.projectDir = value; + fieldSetFlags()[3] = true; + return this; + } + + /** + * Checks whether the 'projectDir' field has been set. + * + * @return True if the 'projectDir' field has been set, false otherwise. + */ + public boolean hasProjectDir() { + return fieldSetFlags()[3]; + } + + /** + * Clears the value of the 'projectDir' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearProjectDir() { + projectDir = null; + fieldSetFlags()[3] = false; + return this; + } + + /** + * Gets the value of the 'workDir' field. + * + * @return The value. + */ + public java.lang.String getWorkDir() { + return workDir; + } + + /** + * Sets the value of the 'workDir' field. + * + * @param value The value of 'workDir'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setWorkDir( + java.lang.String value) { + validate(fields()[4], value); + this.workDir = value; + fieldSetFlags()[4] = true; + return this; + } + + /** + * Checks whether the 'workDir' field has been set. + * + * @return True if the 'workDir' field has been set, false otherwise. + */ + public boolean hasWorkDir() { + return fieldSetFlags()[4]; + } + + /** + * Clears the value of the 'workDir' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearWorkDir() { + workDir = null; + fieldSetFlags()[4] = false; + return this; + } + + /** + * Gets the value of the 'latest' field. + * + * @return The value. + */ + public java.lang.Boolean getLatest() { + return latest; + } + + /** + * Sets the value of the 'latest' field. + * + * @param value The value of 'latest'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setLatest( + java.lang.Boolean value) { + validate(fields()[5], value); + this.latest = value; + fieldSetFlags()[5] = true; + return this; + } + + /** + * Checks whether the 'latest' field has been set. + * + * @return True if the 'latest' field has been set, false otherwise. + */ + public boolean hasLatest() { + return fieldSetFlags()[5]; + } + + /** + * Clears the value of the 'latest' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearLatest() { + latest = null; + fieldSetFlags()[5] = false; + return this; + } + + /** + * Gets the value of the 'resume' field. + * + * @return The value. + */ + public java.lang.String getResume() { + return resume; + } + + /** + * Sets the value of the 'resume' field. + * + * @param value The value of 'resume'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setResume( + java.lang.String value) { + validate(fields()[6], value); + this.resume = value; + fieldSetFlags()[6] = true; + return this; + } + + /** + * Checks whether the 'resume' field has been set. + * + * @return True if the 'resume' field has been set, false otherwise. + */ + public boolean hasResume() { + return fieldSetFlags()[6]; + } + + /** + * Clears the value of the 'resume' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearResume() { + resume = null; + fieldSetFlags()[6] = false; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public EngineParams build() { + try { + EngineParams record = new EngineParams(); + record.defaultContainer = + fieldSetFlags()[0] + ? this.defaultContainer + : (java.lang.String) defaultValue(fields()[0]); + record.revision = + fieldSetFlags()[1] ? this.revision : (java.lang.String) defaultValue(fields()[1]); + record.launchDir = + fieldSetFlags()[2] ? this.launchDir : (java.lang.String) defaultValue(fields()[2]); + record.projectDir = + fieldSetFlags()[3] ? this.projectDir : (java.lang.String) defaultValue(fields()[3]); + record.workDir = + fieldSetFlags()[4] ? this.workDir : (java.lang.String) defaultValue(fields()[4]); + record.latest = + fieldSetFlags()[5] ? this.latest : (java.lang.Boolean) defaultValue(fields()[5]); + record.resume = + fieldSetFlags()[6] ? this.resume : (java.lang.String) defaultValue(fields()[6]); + return record; + } catch (org.apache.avro.AvroMissingFieldException e) { + throw e; + } catch (java.lang.Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumWriter WRITER$ = + (org.apache.avro.io.DatumWriter) MODEL$.createDatumWriter(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException { + WRITER$.write(this, SpecificData.getEncoder(out)); + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumReader READER$ = + (org.apache.avro.io.DatumReader) MODEL$.createDatumReader(SCHEMA$); + + @Override + public void readExternal(java.io.ObjectInput in) throws java.io.IOException { + READER$.read(this, SpecificData.getDecoder(in)); + } + + @Override + protected boolean hasCustomCoders() { + return true; + } + + @Override + public void customEncode(org.apache.avro.io.Encoder out) throws java.io.IOException { + if (this.defaultContainer == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.defaultContainer); + } + + if (this.revision == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.revision); + } + + if (this.launchDir == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.launchDir); + } + + if (this.projectDir == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.projectDir); + } + + if (this.workDir == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.workDir); + } + + if (this.latest == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeBoolean(this.latest); + } + + if (this.resume == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.resume); + } + } + + @Override + public void customDecode(org.apache.avro.io.ResolvingDecoder in) throws java.io.IOException { + org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff(); + if (fieldOrder == null) { + if (in.readIndex() != 1) { + in.readNull(); + this.defaultContainer = null; + } else { + this.defaultContainer = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.revision = null; + } else { + this.revision = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.launchDir = null; + } else { + this.launchDir = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.projectDir = null; + } else { + this.projectDir = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.workDir = null; + } else { + this.workDir = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.latest = null; + } else { + this.latest = in.readBoolean(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.resume = null; + } else { + this.resume = in.readString(); + } + + } else { + for (int i = 0; i < 7; i++) { + switch (fieldOrder[i].pos()) { + case 0: + if (in.readIndex() != 1) { + in.readNull(); + this.defaultContainer = null; + } else { + this.defaultContainer = in.readString(); + } + break; + + case 1: + if (in.readIndex() != 1) { + in.readNull(); + this.revision = null; + } else { + this.revision = in.readString(); + } + break; + + case 2: + if (in.readIndex() != 1) { + in.readNull(); + this.launchDir = null; + } else { + this.launchDir = in.readString(); + } + break; + + case 3: + if (in.readIndex() != 1) { + in.readNull(); + this.projectDir = null; + } else { + this.projectDir = in.readString(); + } + break; + + case 4: + if (in.readIndex() != 1) { + in.readNull(); + this.workDir = null; + } else { + this.workDir = in.readString(); + } + break; + + case 5: + if (in.readIndex() != 1) { + in.readNull(); + this.latest = null; + } else { + this.latest = in.readBoolean(); + } + break; + + case 6: + if (in.readIndex() != 1) { + in.readNull(); + this.resume = null; + } else { + this.resume = in.readString(); + } + break; + + default: + throw new java.io.IOException("Corrupt ResolvingDecoder."); + } + } + } + } +} diff --git a/src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java b/src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java new file mode 100644 index 0000000..ce47619 --- /dev/null +++ b/src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java @@ -0,0 +1,32 @@ +/** + * Autogenerated by Avro + * + *

DO NOT EDIT DIRECTLY + */ +package org.icgc.argo.workflow_management.streams.schema; + +@org.apache.avro.specific.AvroGenerated +public enum RunState implements org.apache.avro.generic.GenericEnumSymbol { + UNKNOWN, + QUEUED, + INITIALIZING, + RUNNING, + PAUSED, + CANCELING, + CANCELED, + COMPLETE, + EXECUTOR_ERROR, + SYSTEM_ERROR; + public static final org.apache.avro.Schema SCHEMA$ = + new org.apache.avro.Schema.Parser() + .parse( + "{\"type\":\"enum\",\"name\":\"RunState\",\"namespace\":\"org.icgc.argo.workflow_management.streams.schema\",\"symbols\":[\"UNKNOWN\",\"QUEUED\",\"INITIALIZING\",\"RUNNING\",\"PAUSED\",\"CANCELING\",\"CANCELED\",\"COMPLETE\",\"EXECUTOR_ERROR\",\"SYSTEM_ERROR\"]}"); + + public static org.apache.avro.Schema getClassSchema() { + return SCHEMA$; + } + + public org.apache.avro.Schema getSchema() { + return SCHEMA$; + } +} diff --git a/src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java b/src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java new file mode 100644 index 0000000..305efd4 --- /dev/null +++ b/src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java @@ -0,0 +1,1115 @@ +/** + * Autogenerated by Avro + * + *

DO NOT EDIT DIRECTLY + */ +package org.icgc.argo.workflow_management.streams.schema; + +import org.apache.avro.message.BinaryMessageDecoder; +import org.apache.avro.message.BinaryMessageEncoder; +import org.apache.avro.message.SchemaStore; +import org.apache.avro.specific.SpecificData; + +@org.apache.avro.specific.AvroGenerated +public class WfMgmtRunMsg extends org.apache.avro.specific.SpecificRecordBase + implements org.apache.avro.specific.SpecificRecord { + private static final long serialVersionUID = -2804506531583278262L; + public static final org.apache.avro.Schema SCHEMA$ = + new org.apache.avro.Schema.Parser() + .parse( + "{\"type\":\"record\",\"name\":\"WfMgmtRunMsg\",\"namespace\":\"org.icgc.argo.workflow_management.streams.schema\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"long\",\"logicalType\":\"date\"},{\"name\":\"runId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"state\",\"type\":{\"type\":\"enum\",\"name\":\"RunState\",\"symbols\":[\"UNKNOWN\",\"QUEUED\",\"INITIALIZING\",\"RUNNING\",\"PAUSED\",\"CANCELING\",\"CANCELED\",\"COMPLETE\",\"EXECUTOR_ERROR\",\"SYSTEM_ERROR\"]}},{\"name\":\"workflowUrl\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowType\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowTypeVersion\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowParamsJsonStr\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowEngineParams\",\"type\":{\"type\":\"record\",\"name\":\"EngineParams\",\"fields\":[{\"name\":\"defaultContainer\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"revision\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"launchDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"projectDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"latest\",\"type\":[\"null\",\"boolean\"],\"default\":null},{\"name\":\"resume\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null}]}}]}"); + + public static org.apache.avro.Schema getClassSchema() { + return SCHEMA$; + } + + private static SpecificData MODEL$ = new SpecificData(); + + private static final BinaryMessageEncoder ENCODER = + new BinaryMessageEncoder(MODEL$, SCHEMA$); + + private static final BinaryMessageDecoder DECODER = + new BinaryMessageDecoder(MODEL$, SCHEMA$); + + /** + * Return the BinaryMessageEncoder instance used by this class. + * + * @return the message encoder used by this class + */ + public static BinaryMessageEncoder getEncoder() { + return ENCODER; + } + + /** + * Return the BinaryMessageDecoder instance used by this class. + * + * @return the message decoder used by this class + */ + public static BinaryMessageDecoder getDecoder() { + return DECODER; + } + + /** + * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link + * SchemaStore}. + * + * @param resolver a {@link SchemaStore} used to find schemas by fingerprint + * @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore + */ + public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { + return new BinaryMessageDecoder(MODEL$, SCHEMA$, resolver); + } + + /** + * Serializes this WfMgmtRunMsg to a ByteBuffer. + * + * @return a buffer holding the serialized data for this instance + * @throws java.io.IOException if this instance could not be serialized + */ + public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { + return ENCODER.encode(this); + } + + /** + * Deserializes a WfMgmtRunMsg from a ByteBuffer. + * + * @param b a byte buffer holding serialized data for an instance of this class + * @return a WfMgmtRunMsg instance decoded from the given buffer + * @throws java.io.IOException if the given bytes could not be deserialized into an instance of + * this class + */ + public static WfMgmtRunMsg fromByteBuffer(java.nio.ByteBuffer b) throws java.io.IOException { + return DECODER.decode(b); + } + + private long timestamp; + private java.lang.String runId; + private org.icgc.argo.workflow_management.streams.schema.RunState state; + private java.lang.String workflowUrl; + private java.lang.String workflowType; + private java.lang.String workflowTypeVersion; + private java.lang.String workflowParamsJsonStr; + private org.icgc.argo.workflow_management.streams.schema.EngineParams workflowEngineParams; + + /** + * Default constructor. Note that this does not initialize fields to their default values from the + * schema. If that is desired then one should use newBuilder(). + */ + public WfMgmtRunMsg() {} + + /** + * All-args constructor. + * + * @param timestamp The new value for timestamp + * @param runId The new value for runId + * @param state The new value for state + * @param workflowUrl The new value for workflowUrl + * @param workflowType The new value for workflowType + * @param workflowTypeVersion The new value for workflowTypeVersion + * @param workflowParamsJsonStr The new value for workflowParamsJsonStr + * @param workflowEngineParams The new value for workflowEngineParams + */ + public WfMgmtRunMsg( + java.lang.Long timestamp, + java.lang.String runId, + org.icgc.argo.workflow_management.streams.schema.RunState state, + java.lang.String workflowUrl, + java.lang.String workflowType, + java.lang.String workflowTypeVersion, + java.lang.String workflowParamsJsonStr, + org.icgc.argo.workflow_management.streams.schema.EngineParams workflowEngineParams) { + this.timestamp = timestamp; + this.runId = runId; + this.state = state; + this.workflowUrl = workflowUrl; + this.workflowType = workflowType; + this.workflowTypeVersion = workflowTypeVersion; + this.workflowParamsJsonStr = workflowParamsJsonStr; + this.workflowEngineParams = workflowEngineParams; + } + + public org.apache.avro.specific.SpecificData getSpecificData() { + return MODEL$; + } + + public org.apache.avro.Schema getSchema() { + return SCHEMA$; + } + // Used by DatumWriter. Applications should not call. + public java.lang.Object get(int field$) { + switch (field$) { + case 0: + return timestamp; + case 1: + return runId; + case 2: + return state; + case 3: + return workflowUrl; + case 4: + return workflowType; + case 5: + return workflowTypeVersion; + case 6: + return workflowParamsJsonStr; + case 7: + return workflowEngineParams; + default: + throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + // Used by DatumReader. Applications should not call. + @SuppressWarnings(value = "unchecked") + public void put(int field$, java.lang.Object value$) { + switch (field$) { + case 0: + timestamp = (java.lang.Long) value$; + break; + case 1: + runId = value$ != null ? value$.toString() : null; + break; + case 2: + state = (org.icgc.argo.workflow_management.streams.schema.RunState) value$; + break; + case 3: + workflowUrl = value$ != null ? value$.toString() : null; + break; + case 4: + workflowType = value$ != null ? value$.toString() : null; + break; + case 5: + workflowTypeVersion = value$ != null ? value$.toString() : null; + break; + case 6: + workflowParamsJsonStr = value$ != null ? value$.toString() : null; + break; + case 7: + workflowEngineParams = + (org.icgc.argo.workflow_management.streams.schema.EngineParams) value$; + break; + default: + throw new IndexOutOfBoundsException("Invalid index: " + field$); + } + } + + /** + * Gets the value of the 'timestamp' field. + * + * @return The value of the 'timestamp' field. + */ + public long getTimestamp() { + return timestamp; + } + + /** + * Sets the value of the 'timestamp' field. + * + * @param value the value to set. + */ + public void setTimestamp(long value) { + this.timestamp = value; + } + + /** + * Gets the value of the 'runId' field. + * + * @return The value of the 'runId' field. + */ + public java.lang.String getRunId() { + return runId; + } + + /** + * Sets the value of the 'runId' field. + * + * @param value the value to set. + */ + public void setRunId(java.lang.String value) { + this.runId = value; + } + + /** + * Gets the value of the 'state' field. + * + * @return The value of the 'state' field. + */ + public org.icgc.argo.workflow_management.streams.schema.RunState getState() { + return state; + } + + /** + * Sets the value of the 'state' field. + * + * @param value the value to set. + */ + public void setState(org.icgc.argo.workflow_management.streams.schema.RunState value) { + this.state = value; + } + + /** + * Gets the value of the 'workflowUrl' field. + * + * @return The value of the 'workflowUrl' field. + */ + public java.lang.String getWorkflowUrl() { + return workflowUrl; + } + + /** + * Sets the value of the 'workflowUrl' field. + * + * @param value the value to set. + */ + public void setWorkflowUrl(java.lang.String value) { + this.workflowUrl = value; + } + + /** + * Gets the value of the 'workflowType' field. + * + * @return The value of the 'workflowType' field. + */ + public java.lang.String getWorkflowType() { + return workflowType; + } + + /** + * Sets the value of the 'workflowType' field. + * + * @param value the value to set. + */ + public void setWorkflowType(java.lang.String value) { + this.workflowType = value; + } + + /** + * Gets the value of the 'workflowTypeVersion' field. + * + * @return The value of the 'workflowTypeVersion' field. + */ + public java.lang.String getWorkflowTypeVersion() { + return workflowTypeVersion; + } + + /** + * Sets the value of the 'workflowTypeVersion' field. + * + * @param value the value to set. + */ + public void setWorkflowTypeVersion(java.lang.String value) { + this.workflowTypeVersion = value; + } + + /** + * Gets the value of the 'workflowParamsJsonStr' field. + * + * @return The value of the 'workflowParamsJsonStr' field. + */ + public java.lang.String getWorkflowParamsJsonStr() { + return workflowParamsJsonStr; + } + + /** + * Sets the value of the 'workflowParamsJsonStr' field. + * + * @param value the value to set. + */ + public void setWorkflowParamsJsonStr(java.lang.String value) { + this.workflowParamsJsonStr = value; + } + + /** + * Gets the value of the 'workflowEngineParams' field. + * + * @return The value of the 'workflowEngineParams' field. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams getWorkflowEngineParams() { + return workflowEngineParams; + } + + /** + * Sets the value of the 'workflowEngineParams' field. + * + * @param value the value to set. + */ + public void setWorkflowEngineParams( + org.icgc.argo.workflow_management.streams.schema.EngineParams value) { + this.workflowEngineParams = value; + } + + /** + * Creates a new WfMgmtRunMsg RecordBuilder. + * + * @return A new WfMgmtRunMsg RecordBuilder + */ + public static org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder newBuilder() { + return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(); + } + + /** + * Creates a new WfMgmtRunMsg RecordBuilder by copying an existing Builder. + * + * @param other The existing builder to copy. + * @return A new WfMgmtRunMsg RecordBuilder + */ + public static org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder newBuilder( + org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder other) { + if (other == null) { + return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(); + } else { + return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(other); + } + } + + /** + * Creates a new WfMgmtRunMsg RecordBuilder by copying an existing WfMgmtRunMsg instance. + * + * @param other The existing instance to copy. + * @return A new WfMgmtRunMsg RecordBuilder + */ + public static org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder newBuilder( + org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg other) { + if (other == null) { + return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(); + } else { + return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(other); + } + } + + /** RecordBuilder for WfMgmtRunMsg instances. */ + @org.apache.avro.specific.AvroGenerated + public static class Builder + extends org.apache.avro.specific.SpecificRecordBuilderBase + implements org.apache.avro.data.RecordBuilder { + + private long timestamp; + private java.lang.String runId; + private org.icgc.argo.workflow_management.streams.schema.RunState state; + private java.lang.String workflowUrl; + private java.lang.String workflowType; + private java.lang.String workflowTypeVersion; + private java.lang.String workflowParamsJsonStr; + private org.icgc.argo.workflow_management.streams.schema.EngineParams workflowEngineParams; + private org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder + workflowEngineParamsBuilder; + + /** Creates a new Builder */ + private Builder() { + super(SCHEMA$); + } + + /** + * Creates a Builder by copying an existing Builder. + * + * @param other The existing Builder to copy. + */ + private Builder(org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder other) { + super(other); + if (isValidValue(fields()[0], other.timestamp)) { + this.timestamp = data().deepCopy(fields()[0].schema(), other.timestamp); + fieldSetFlags()[0] = other.fieldSetFlags()[0]; + } + if (isValidValue(fields()[1], other.runId)) { + this.runId = data().deepCopy(fields()[1].schema(), other.runId); + fieldSetFlags()[1] = other.fieldSetFlags()[1]; + } + if (isValidValue(fields()[2], other.state)) { + this.state = data().deepCopy(fields()[2].schema(), other.state); + fieldSetFlags()[2] = other.fieldSetFlags()[2]; + } + if (isValidValue(fields()[3], other.workflowUrl)) { + this.workflowUrl = data().deepCopy(fields()[3].schema(), other.workflowUrl); + fieldSetFlags()[3] = other.fieldSetFlags()[3]; + } + if (isValidValue(fields()[4], other.workflowType)) { + this.workflowType = data().deepCopy(fields()[4].schema(), other.workflowType); + fieldSetFlags()[4] = other.fieldSetFlags()[4]; + } + if (isValidValue(fields()[5], other.workflowTypeVersion)) { + this.workflowTypeVersion = data().deepCopy(fields()[5].schema(), other.workflowTypeVersion); + fieldSetFlags()[5] = other.fieldSetFlags()[5]; + } + if (isValidValue(fields()[6], other.workflowParamsJsonStr)) { + this.workflowParamsJsonStr = + data().deepCopy(fields()[6].schema(), other.workflowParamsJsonStr); + fieldSetFlags()[6] = other.fieldSetFlags()[6]; + } + if (isValidValue(fields()[7], other.workflowEngineParams)) { + this.workflowEngineParams = + data().deepCopy(fields()[7].schema(), other.workflowEngineParams); + fieldSetFlags()[7] = other.fieldSetFlags()[7]; + } + if (other.hasWorkflowEngineParamsBuilder()) { + this.workflowEngineParamsBuilder = + org.icgc.argo.workflow_management.streams.schema.EngineParams.newBuilder( + other.getWorkflowEngineParamsBuilder()); + } + } + + /** + * Creates a Builder by copying an existing WfMgmtRunMsg instance + * + * @param other The existing instance to copy. + */ + private Builder(org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg other) { + super(SCHEMA$); + if (isValidValue(fields()[0], other.timestamp)) { + this.timestamp = data().deepCopy(fields()[0].schema(), other.timestamp); + fieldSetFlags()[0] = true; + } + if (isValidValue(fields()[1], other.runId)) { + this.runId = data().deepCopy(fields()[1].schema(), other.runId); + fieldSetFlags()[1] = true; + } + if (isValidValue(fields()[2], other.state)) { + this.state = data().deepCopy(fields()[2].schema(), other.state); + fieldSetFlags()[2] = true; + } + if (isValidValue(fields()[3], other.workflowUrl)) { + this.workflowUrl = data().deepCopy(fields()[3].schema(), other.workflowUrl); + fieldSetFlags()[3] = true; + } + if (isValidValue(fields()[4], other.workflowType)) { + this.workflowType = data().deepCopy(fields()[4].schema(), other.workflowType); + fieldSetFlags()[4] = true; + } + if (isValidValue(fields()[5], other.workflowTypeVersion)) { + this.workflowTypeVersion = data().deepCopy(fields()[5].schema(), other.workflowTypeVersion); + fieldSetFlags()[5] = true; + } + if (isValidValue(fields()[6], other.workflowParamsJsonStr)) { + this.workflowParamsJsonStr = + data().deepCopy(fields()[6].schema(), other.workflowParamsJsonStr); + fieldSetFlags()[6] = true; + } + if (isValidValue(fields()[7], other.workflowEngineParams)) { + this.workflowEngineParams = + data().deepCopy(fields()[7].schema(), other.workflowEngineParams); + fieldSetFlags()[7] = true; + } + this.workflowEngineParamsBuilder = null; + } + + /** + * Gets the value of the 'timestamp' field. + * + * @return The value. + */ + public long getTimestamp() { + return timestamp; + } + + /** + * Sets the value of the 'timestamp' field. + * + * @param value The value of 'timestamp'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setTimestamp( + long value) { + validate(fields()[0], value); + this.timestamp = value; + fieldSetFlags()[0] = true; + return this; + } + + /** + * Checks whether the 'timestamp' field has been set. + * + * @return True if the 'timestamp' field has been set, false otherwise. + */ + public boolean hasTimestamp() { + return fieldSetFlags()[0]; + } + + /** + * Clears the value of the 'timestamp' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder clearTimestamp() { + fieldSetFlags()[0] = false; + return this; + } + + /** + * Gets the value of the 'runId' field. + * + * @return The value. + */ + public java.lang.String getRunId() { + return runId; + } + + /** + * Sets the value of the 'runId' field. + * + * @param value The value of 'runId'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setRunId( + java.lang.String value) { + validate(fields()[1], value); + this.runId = value; + fieldSetFlags()[1] = true; + return this; + } + + /** + * Checks whether the 'runId' field has been set. + * + * @return True if the 'runId' field has been set, false otherwise. + */ + public boolean hasRunId() { + return fieldSetFlags()[1]; + } + + /** + * Clears the value of the 'runId' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder clearRunId() { + runId = null; + fieldSetFlags()[1] = false; + return this; + } + + /** + * Gets the value of the 'state' field. + * + * @return The value. + */ + public org.icgc.argo.workflow_management.streams.schema.RunState getState() { + return state; + } + + /** + * Sets the value of the 'state' field. + * + * @param value The value of 'state'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setState( + org.icgc.argo.workflow_management.streams.schema.RunState value) { + validate(fields()[2], value); + this.state = value; + fieldSetFlags()[2] = true; + return this; + } + + /** + * Checks whether the 'state' field has been set. + * + * @return True if the 'state' field has been set, false otherwise. + */ + public boolean hasState() { + return fieldSetFlags()[2]; + } + + /** + * Clears the value of the 'state' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder clearState() { + state = null; + fieldSetFlags()[2] = false; + return this; + } + + /** + * Gets the value of the 'workflowUrl' field. + * + * @return The value. + */ + public java.lang.String getWorkflowUrl() { + return workflowUrl; + } + + /** + * Sets the value of the 'workflowUrl' field. + * + * @param value The value of 'workflowUrl'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setWorkflowUrl( + java.lang.String value) { + validate(fields()[3], value); + this.workflowUrl = value; + fieldSetFlags()[3] = true; + return this; + } + + /** + * Checks whether the 'workflowUrl' field has been set. + * + * @return True if the 'workflowUrl' field has been set, false otherwise. + */ + public boolean hasWorkflowUrl() { + return fieldSetFlags()[3]; + } + + /** + * Clears the value of the 'workflowUrl' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + clearWorkflowUrl() { + workflowUrl = null; + fieldSetFlags()[3] = false; + return this; + } + + /** + * Gets the value of the 'workflowType' field. + * + * @return The value. + */ + public java.lang.String getWorkflowType() { + return workflowType; + } + + /** + * Sets the value of the 'workflowType' field. + * + * @param value The value of 'workflowType'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setWorkflowType( + java.lang.String value) { + validate(fields()[4], value); + this.workflowType = value; + fieldSetFlags()[4] = true; + return this; + } + + /** + * Checks whether the 'workflowType' field has been set. + * + * @return True if the 'workflowType' field has been set, false otherwise. + */ + public boolean hasWorkflowType() { + return fieldSetFlags()[4]; + } + + /** + * Clears the value of the 'workflowType' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + clearWorkflowType() { + workflowType = null; + fieldSetFlags()[4] = false; + return this; + } + + /** + * Gets the value of the 'workflowTypeVersion' field. + * + * @return The value. + */ + public java.lang.String getWorkflowTypeVersion() { + return workflowTypeVersion; + } + + /** + * Sets the value of the 'workflowTypeVersion' field. + * + * @param value The value of 'workflowTypeVersion'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + setWorkflowTypeVersion(java.lang.String value) { + validate(fields()[5], value); + this.workflowTypeVersion = value; + fieldSetFlags()[5] = true; + return this; + } + + /** + * Checks whether the 'workflowTypeVersion' field has been set. + * + * @return True if the 'workflowTypeVersion' field has been set, false otherwise. + */ + public boolean hasWorkflowTypeVersion() { + return fieldSetFlags()[5]; + } + + /** + * Clears the value of the 'workflowTypeVersion' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + clearWorkflowTypeVersion() { + workflowTypeVersion = null; + fieldSetFlags()[5] = false; + return this; + } + + /** + * Gets the value of the 'workflowParamsJsonStr' field. + * + * @return The value. + */ + public java.lang.String getWorkflowParamsJsonStr() { + return workflowParamsJsonStr; + } + + /** + * Sets the value of the 'workflowParamsJsonStr' field. + * + * @param value The value of 'workflowParamsJsonStr'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + setWorkflowParamsJsonStr(java.lang.String value) { + validate(fields()[6], value); + this.workflowParamsJsonStr = value; + fieldSetFlags()[6] = true; + return this; + } + + /** + * Checks whether the 'workflowParamsJsonStr' field has been set. + * + * @return True if the 'workflowParamsJsonStr' field has been set, false otherwise. + */ + public boolean hasWorkflowParamsJsonStr() { + return fieldSetFlags()[6]; + } + + /** + * Clears the value of the 'workflowParamsJsonStr' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + clearWorkflowParamsJsonStr() { + workflowParamsJsonStr = null; + fieldSetFlags()[6] = false; + return this; + } + + /** + * Gets the value of the 'workflowEngineParams' field. + * + * @return The value. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams getWorkflowEngineParams() { + return workflowEngineParams; + } + + /** + * Sets the value of the 'workflowEngineParams' field. + * + * @param value The value of 'workflowEngineParams'. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + setWorkflowEngineParams( + org.icgc.argo.workflow_management.streams.schema.EngineParams value) { + validate(fields()[7], value); + this.workflowEngineParamsBuilder = null; + this.workflowEngineParams = value; + fieldSetFlags()[7] = true; + return this; + } + + /** + * Checks whether the 'workflowEngineParams' field has been set. + * + * @return True if the 'workflowEngineParams' field has been set, false otherwise. + */ + public boolean hasWorkflowEngineParams() { + return fieldSetFlags()[7]; + } + + /** + * Gets the Builder instance for the 'workflowEngineParams' field and creates one if it doesn't + * exist yet. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder + getWorkflowEngineParamsBuilder() { + if (workflowEngineParamsBuilder == null) { + if (hasWorkflowEngineParams()) { + setWorkflowEngineParamsBuilder( + org.icgc.argo.workflow_management.streams.schema.EngineParams.newBuilder( + workflowEngineParams)); + } else { + setWorkflowEngineParamsBuilder( + org.icgc.argo.workflow_management.streams.schema.EngineParams.newBuilder()); + } + } + return workflowEngineParamsBuilder; + } + + /** + * Sets the Builder instance for the 'workflowEngineParams' field + * + * @param value The builder instance that must be set. + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + setWorkflowEngineParamsBuilder( + org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder value) { + clearWorkflowEngineParams(); + workflowEngineParamsBuilder = value; + return this; + } + + /** + * Checks whether the 'workflowEngineParams' field has an active Builder instance + * + * @return True if the 'workflowEngineParams' field has an active Builder instance + */ + public boolean hasWorkflowEngineParamsBuilder() { + return workflowEngineParamsBuilder != null; + } + + /** + * Clears the value of the 'workflowEngineParams' field. + * + * @return This builder. + */ + public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder + clearWorkflowEngineParams() { + workflowEngineParams = null; + workflowEngineParamsBuilder = null; + fieldSetFlags()[7] = false; + return this; + } + + @Override + @SuppressWarnings("unchecked") + public WfMgmtRunMsg build() { + try { + WfMgmtRunMsg record = new WfMgmtRunMsg(); + record.timestamp = + fieldSetFlags()[0] ? this.timestamp : (java.lang.Long) defaultValue(fields()[0]); + record.runId = + fieldSetFlags()[1] ? this.runId : (java.lang.String) defaultValue(fields()[1]); + record.state = + fieldSetFlags()[2] + ? this.state + : (org.icgc.argo.workflow_management.streams.schema.RunState) + defaultValue(fields()[2]); + record.workflowUrl = + fieldSetFlags()[3] ? this.workflowUrl : (java.lang.String) defaultValue(fields()[3]); + record.workflowType = + fieldSetFlags()[4] ? this.workflowType : (java.lang.String) defaultValue(fields()[4]); + record.workflowTypeVersion = + fieldSetFlags()[5] + ? this.workflowTypeVersion + : (java.lang.String) defaultValue(fields()[5]); + record.workflowParamsJsonStr = + fieldSetFlags()[6] + ? this.workflowParamsJsonStr + : (java.lang.String) defaultValue(fields()[6]); + if (workflowEngineParamsBuilder != null) { + try { + record.workflowEngineParams = this.workflowEngineParamsBuilder.build(); + } catch (org.apache.avro.AvroMissingFieldException e) { + e.addParentField(record.getSchema().getField("workflowEngineParams")); + throw e; + } + } else { + record.workflowEngineParams = + fieldSetFlags()[7] + ? this.workflowEngineParams + : (org.icgc.argo.workflow_management.streams.schema.EngineParams) + defaultValue(fields()[7]); + } + return record; + } catch (org.apache.avro.AvroMissingFieldException e) { + throw e; + } catch (java.lang.Exception e) { + throw new org.apache.avro.AvroRuntimeException(e); + } + } + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumWriter WRITER$ = + (org.apache.avro.io.DatumWriter) MODEL$.createDatumWriter(SCHEMA$); + + @Override + public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException { + WRITER$.write(this, SpecificData.getEncoder(out)); + } + + @SuppressWarnings("unchecked") + private static final org.apache.avro.io.DatumReader READER$ = + (org.apache.avro.io.DatumReader) MODEL$.createDatumReader(SCHEMA$); + + @Override + public void readExternal(java.io.ObjectInput in) throws java.io.IOException { + READER$.read(this, SpecificData.getDecoder(in)); + } + + @Override + protected boolean hasCustomCoders() { + return true; + } + + @Override + public void customEncode(org.apache.avro.io.Encoder out) throws java.io.IOException { + out.writeLong(this.timestamp); + + out.writeString(this.runId); + + out.writeEnum(this.state.ordinal()); + + if (this.workflowUrl == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.workflowUrl); + } + + if (this.workflowType == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.workflowType); + } + + if (this.workflowTypeVersion == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.workflowTypeVersion); + } + + if (this.workflowParamsJsonStr == null) { + out.writeIndex(0); + out.writeNull(); + } else { + out.writeIndex(1); + out.writeString(this.workflowParamsJsonStr); + } + + this.workflowEngineParams.customEncode(out); + } + + @Override + public void customDecode(org.apache.avro.io.ResolvingDecoder in) throws java.io.IOException { + org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff(); + if (fieldOrder == null) { + this.timestamp = in.readLong(); + + this.runId = in.readString(); + + this.state = + org.icgc.argo.workflow_management.streams.schema.RunState.values()[in.readEnum()]; + + if (in.readIndex() != 1) { + in.readNull(); + this.workflowUrl = null; + } else { + this.workflowUrl = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.workflowType = null; + } else { + this.workflowType = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.workflowTypeVersion = null; + } else { + this.workflowTypeVersion = in.readString(); + } + + if (in.readIndex() != 1) { + in.readNull(); + this.workflowParamsJsonStr = null; + } else { + this.workflowParamsJsonStr = in.readString(); + } + + if (this.workflowEngineParams == null) { + this.workflowEngineParams = + new org.icgc.argo.workflow_management.streams.schema.EngineParams(); + } + this.workflowEngineParams.customDecode(in); + + } else { + for (int i = 0; i < 8; i++) { + switch (fieldOrder[i].pos()) { + case 0: + this.timestamp = in.readLong(); + break; + + case 1: + this.runId = in.readString(); + break; + + case 2: + this.state = + org.icgc.argo.workflow_management.streams.schema.RunState.values()[in.readEnum()]; + break; + + case 3: + if (in.readIndex() != 1) { + in.readNull(); + this.workflowUrl = null; + } else { + this.workflowUrl = in.readString(); + } + break; + + case 4: + if (in.readIndex() != 1) { + in.readNull(); + this.workflowType = null; + } else { + this.workflowType = in.readString(); + } + break; + + case 5: + if (in.readIndex() != 1) { + in.readNull(); + this.workflowTypeVersion = null; + } else { + this.workflowTypeVersion = in.readString(); + } + break; + + case 6: + if (in.readIndex() != 1) { + in.readNull(); + this.workflowParamsJsonStr = null; + } else { + this.workflowParamsJsonStr = in.readString(); + } + break; + + case 7: + if (this.workflowEngineParams == null) { + this.workflowEngineParams = + new org.icgc.argo.workflow_management.streams.schema.EngineParams(); + } + this.workflowEngineParams.customDecode(in); + break; + + default: + throw new java.io.IOException("Corrupt ResolvingDecoder."); + } + } + } + } +} diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java new file mode 100644 index 0000000..0ed53ce --- /dev/null +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -0,0 +1,30 @@ +package org.icgc.argo.workflow_management.util; + +import lombok.NonNull; +import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; + +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class VolumeMounts { + public static List extract(@NonNull List mappings, @NonNull String workDir) { + return mappings.stream() + .filter( + mapping -> + Pattern.compile(Pattern.quote(mapping.split(",")[1])).matcher(workDir).find()) + .map(matchingMapping -> matchingMapping.split(",")[0]) + .collect(Collectors.toList()); + } + + public static List extract( + NextflowProperties.K8sProperties k8sProperties, String workDir) { + return Optional.ofNullable(workDir) + .flatMap( + presentWorkDir -> + Optional.ofNullable(k8sProperties.getVolMountMappings()) + .map(volMountsMapping -> extract(volMountsMapping, workDir))) + .orElse(k8sProperties.getVolMounts()); + } +} diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java index cc8a1e1..bb04c69 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java @@ -46,6 +46,7 @@ import org.icgc.argo.workflow_management.exception.ReflectionUtilsException; import org.icgc.argo.workflow_management.streams.WebLogEventSender; import org.icgc.argo.workflow_management.util.ConditionalPutMap; +import org.icgc.argo.workflow_management.util.VolumeMounts; import org.icgc.argo.workflow_management.wes.model.*; import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; import org.icgc.argo.workflow_management.wes.secret.SecretProvider; @@ -151,13 +152,13 @@ private String startRun(RunParams params) { private DefaultKubernetesClient createWorkflowRunK8sClient() { try { val masterUrl = config.getK8s().getMasterUrl(); - val namespace = config.getK8s().getRunNamespace(); + val runNamespace = config.getK8s().getRunNamespace(); val trustCertificate = config.getK8s().isTrustCertificate(); val config = new ConfigBuilder() .withTrustCerts(trustCertificate) .withMasterUrl(masterUrl) - .withNamespace(namespace) + .withNamespace(runNamespace) .build(); return new DefaultKubernetesClient(config); } catch (KubernetesClientException e) { @@ -256,9 +257,8 @@ private CmdKubeRun createCmd(@NonNull Launcher launcher, @NonNull RunParams para cmdParams.put("args", List.of(params.getWorkflowUrl())); cmdParams.put("paramsFile", createParamsFile(runName, params.getWorkflowParams())); - // K8s options from application.yml + // Namespace that workflow-management is operating in cmdParams.put("namespace", k8sConfig.getNamespace()); - cmdParams.put("volMounts", k8sConfig.getVolMounts()); // Where to POST event-based logging cmdParams.put("withWebLog", webLogUrl); @@ -324,6 +324,9 @@ private CmdKubeRun createCmd(@NonNull Launcher launcher, @NonNull RunParams para cmdParams.put("process", processOptions); } + // volume mount config for run + cmdParams.put("volMounts", VolumeMounts.extract(k8sConfig, workflowEngineParams.getWorkDir())); + return createWithReflection(CmdKubeRun.class, cmdParams) .orElseThrow(ReflectionUtilsException::new); } diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index a17093f..a9c3014 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -20,6 +20,7 @@ import java.util.List; import lombok.Data; +import lombok.NonNull; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; @@ -41,7 +42,8 @@ public static class K8sProperties { private String serviceAccount; private String namespace; private String runNamespace; - private List volMounts; + @NonNull private List volMounts; + private List volMountMappings; private String masterUrl; private boolean trustCertificate; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e10c223..e457050 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -6,6 +6,8 @@ nextflow: serviceAccount: "default" volMounts: - "pv-claim:/some/dir" + volMountMappings: + - "pv-claim:/some/dir,/some/dir" masterUrl: "localhost:8080" trustCertificate: false weblogUrl: "http://localhost" diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java new file mode 100644 index 0000000..2e18c77 --- /dev/null +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -0,0 +1,68 @@ +package org.icgc.argo.workflow_management; + +import lombok.val; +import org.icgc.argo.workflow_management.util.VolumeMounts; +import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; +import org.junit.Test; +import static org.junit.Assert.assertEquals; +import java.util.List; + +public class VolumeMountsTest { + + @Test + public void defaultConfigTest() { + val mappings = List.of("pv-claim:/some/dir,/some/dir/"); + + assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(mappings, "/some/dir/with/sub/dir")); + } + + @Test + public void multiDirTest() { + val mappings = List.of("pv-claim-one:/test-dir-1/,/dir-one/", "pv-claim-two:/test-dir-2/,/dir-two/", "pv-claim-three:/test-dir-3/,/dir-three/"); + + assertEquals( + List.of("pv-claim-one:/test-dir-1/"), + VolumeMounts.extract(mappings, "/dir-one/test/dir")); + + assertEquals( + List.of("pv-claim-two:/test-dir-2/"), + VolumeMounts.extract(mappings, "/dir-two/test/dir")); + + assertEquals( + List.of("pv-claim-three:/test-dir-3/"), + VolumeMounts.extract(mappings, "/dir-three/test/dir")); + } + + @Test + public void multiMatchTest() { + val mappings = List.of("pv-claim-one:/test-dir-1/,/vol-dir/", "pv-claim-one:/test-dir-2/,/vol-dir/", "pv-claim-one:/test-dir-3/,/vol-dir/"); + + assertEquals(List.of("pv-claim-one:/test-dir-1/", "pv-claim-one:/test-dir-2/", "pv-claim-one:/test-dir-3/"), VolumeMounts.extract(mappings, "/vol-dir/sub/dir")); + } + + @Test + public void testExtractFromPropertiesWhenPresent() { + val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); + k8sProperties.setVolMountMappings(List.of("pv-claim:/some/dir,/some/dir/")); + + assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); + } + + @Test + public void testExtractFromPropertiesWhenMissingVolMountsMapping() { + val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); + k8sProperties.setVolMounts(List.of("pv-claim:/some/dir")); + + assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); + } + + @Test + public void testExtractFromPropertiesWhenMissingWorkdir() { + val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); + k8sProperties.setVolMountMappings(List.of("pv-claim:/some/dir,/some/dir/")); + k8sProperties.setVolMounts(List.of("pv-claim:/some/dir")); + + assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, null)); + } + +} From c22648158da695c256228e5667dd8021eb825160 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 23 Aug 2021 15:26:18 -0400 Subject: [PATCH 03/26] imports + mvn fmt --- .../util/VolumeMounts.java | 5 +-- .../workflow_management/VolumeMountsTest.java | 44 ++++++++++++------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 0ed53ce..f648df3 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,12 +1,11 @@ package org.icgc.argo.workflow_management.util; -import lombok.NonNull; -import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; - import java.util.List; import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Collectors; +import lombok.NonNull; +import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { public static List extract(@NonNull List mappings, @NonNull String workDir) { diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index 2e18c77..105aae0 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -1,11 +1,12 @@ package org.icgc.argo.workflow_management; +import static org.junit.Assert.assertEquals; + +import java.util.List; import lombok.val; import org.icgc.argo.workflow_management.util.VolumeMounts; import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import java.util.List; public class VolumeMountsTest { @@ -13,31 +14,41 @@ public class VolumeMountsTest { public void defaultConfigTest() { val mappings = List.of("pv-claim:/some/dir,/some/dir/"); - assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(mappings, "/some/dir/with/sub/dir")); + assertEquals( + List.of("pv-claim:/some/dir"), VolumeMounts.extract(mappings, "/some/dir/with/sub/dir")); } @Test public void multiDirTest() { - val mappings = List.of("pv-claim-one:/test-dir-1/,/dir-one/", "pv-claim-two:/test-dir-2/,/dir-two/", "pv-claim-three:/test-dir-3/,/dir-three/"); + val mappings = + List.of( + "pv-claim-one:/test-dir-1/,/dir-one/", + "pv-claim-two:/test-dir-2/,/dir-two/", + "pv-claim-three:/test-dir-3/,/dir-three/"); assertEquals( - List.of("pv-claim-one:/test-dir-1/"), - VolumeMounts.extract(mappings, "/dir-one/test/dir")); + List.of("pv-claim-one:/test-dir-1/"), VolumeMounts.extract(mappings, "/dir-one/test/dir")); assertEquals( - List.of("pv-claim-two:/test-dir-2/"), - VolumeMounts.extract(mappings, "/dir-two/test/dir")); + List.of("pv-claim-two:/test-dir-2/"), VolumeMounts.extract(mappings, "/dir-two/test/dir")); assertEquals( - List.of("pv-claim-three:/test-dir-3/"), - VolumeMounts.extract(mappings, "/dir-three/test/dir")); + List.of("pv-claim-three:/test-dir-3/"), + VolumeMounts.extract(mappings, "/dir-three/test/dir")); } @Test public void multiMatchTest() { - val mappings = List.of("pv-claim-one:/test-dir-1/,/vol-dir/", "pv-claim-one:/test-dir-2/,/vol-dir/", "pv-claim-one:/test-dir-3/,/vol-dir/"); + val mappings = + List.of( + "pv-claim-one:/test-dir-1/,/vol-dir/", + "pv-claim-one:/test-dir-2/,/vol-dir/", + "pv-claim-one:/test-dir-3/,/vol-dir/"); - assertEquals(List.of("pv-claim-one:/test-dir-1/", "pv-claim-one:/test-dir-2/", "pv-claim-one:/test-dir-3/"), VolumeMounts.extract(mappings, "/vol-dir/sub/dir")); + assertEquals( + List.of( + "pv-claim-one:/test-dir-1/", "pv-claim-one:/test-dir-2/", "pv-claim-one:/test-dir-3/"), + VolumeMounts.extract(mappings, "/vol-dir/sub/dir")); } @Test @@ -45,7 +56,9 @@ public void testExtractFromPropertiesWhenPresent() { val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); k8sProperties.setVolMountMappings(List.of("pv-claim:/some/dir,/some/dir/")); - assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); + assertEquals( + List.of("pv-claim:/some/dir"), + VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); } @Test @@ -53,7 +66,9 @@ public void testExtractFromPropertiesWhenMissingVolMountsMapping() { val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); k8sProperties.setVolMounts(List.of("pv-claim:/some/dir")); - assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); + assertEquals( + List.of("pv-claim:/some/dir"), + VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); } @Test @@ -64,5 +79,4 @@ public void testExtractFromPropertiesWhenMissingWorkdir() { assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, null)); } - } From 16037c4e5c06765cd2ea137f4a9cafb6af34175a Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 23 Aug 2021 16:17:10 -0400 Subject: [PATCH 04/26] update to include launch and project dirs --- .../util/VolumeMounts.java | 37 +++++--- .../wes/NextflowService.java | 2 +- .../wes/properties/NextflowProperties.java | 6 +- .../workflow_management/VolumeMountsTest.java | 95 ++++++++++++++----- 4 files changed, 97 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index f648df3..26e2270 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,29 +1,40 @@ package org.icgc.argo.workflow_management.util; -import java.util.List; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.NonNull; +import lombok.val; +import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { - public static List extract(@NonNull List mappings, @NonNull String workDir) { + public static Set extract(@NonNull Set mappings, @NonNull String path) { return mappings.stream() .filter( - mapping -> - Pattern.compile(Pattern.quote(mapping.split(",")[1])).matcher(workDir).find()) + mapping -> Pattern.compile(Pattern.quote(mapping.split(",")[1])).matcher(path).find()) .map(matchingMapping -> matchingMapping.split(",")[0]) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); } - public static List extract( - NextflowProperties.K8sProperties k8sProperties, String workDir) { - return Optional.ofNullable(workDir) - .flatMap( - presentWorkDir -> - Optional.ofNullable(k8sProperties.getVolMountMappings()) - .map(volMountsMapping -> extract(volMountsMapping, workDir))) - .orElse(k8sProperties.getVolMounts()); + public static Set extract( + NextflowProperties.K8sProperties k8sProperties, WorkflowEngineParams workflowEngineParams) { + val volMounts = + Stream.of( + workflowEngineParams.getLaunchDir(), + workflowEngineParams.getProjectDir(), + workflowEngineParams.getWorkDir()) + .filter(Objects::nonNull) + .flatMap( + path -> + Optional.ofNullable(k8sProperties.getVolMountMappings()) + .map(volMountsMapping -> extract(volMountsMapping, path)) + .orElse(Set.copyOf(k8sProperties.getVolMounts())).stream()) + .collect(Collectors.toSet()); + + return volMounts.isEmpty() ? k8sProperties.getVolMounts() : volMounts; } } diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java index bb04c69..8f50891 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java @@ -325,7 +325,7 @@ private CmdKubeRun createCmd(@NonNull Launcher launcher, @NonNull RunParams para } // volume mount config for run - cmdParams.put("volMounts", VolumeMounts.extract(k8sConfig, workflowEngineParams.getWorkDir())); + cmdParams.put("volMounts", VolumeMounts.extract(k8sConfig, workflowEngineParams)); return createWithReflection(CmdKubeRun.class, cmdParams) .orElseThrow(ReflectionUtilsException::new); diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index a9c3014..ef0ebda 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -18,7 +18,7 @@ package org.icgc.argo.workflow_management.wes.properties; -import java.util.List; +import java.util.Set; import lombok.Data; import lombok.NonNull; import org.springframework.boot.context.properties.ConfigurationProperties; @@ -42,8 +42,8 @@ public static class K8sProperties { private String serviceAccount; private String namespace; private String runNamespace; - @NonNull private List volMounts; - private List volMountMappings; + @NonNull private Set volMounts; + private Set volMountMappings; private String masterUrl; private boolean trustCertificate; } diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index 105aae0..c44074c 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -2,9 +2,10 @@ import static org.junit.Assert.assertEquals; -import java.util.List; +import java.util.Set; import lombok.val; import org.icgc.argo.workflow_management.util.VolumeMounts; +import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; import org.junit.Test; @@ -12,71 +13,113 @@ public class VolumeMountsTest { @Test public void defaultConfigTest() { - val mappings = List.of("pv-claim:/some/dir,/some/dir/"); + val mappings = Set.of("pv-claim:/some/dir,/some/dir/"); assertEquals( - List.of("pv-claim:/some/dir"), VolumeMounts.extract(mappings, "/some/dir/with/sub/dir")); + Set.of("pv-claim:/some/dir"), VolumeMounts.extract(mappings, "/some/dir/with/sub/dir")); } @Test public void multiDirTest() { + // include some inconsistent matching strings (with and without /) val mappings = - List.of( - "pv-claim-one:/test-dir-1/,/dir-one/", - "pv-claim-two:/test-dir-2/,/dir-two/", - "pv-claim-three:/test-dir-3/,/dir-three/"); + Set.of( + "pv-claim-one:/test-dir-1/,dir-one", + "pv-claim-two:/test-dir-2/,/dir-two", + "pv-claim-three:/test-dir-3/,dir-three/"); assertEquals( - List.of("pv-claim-one:/test-dir-1/"), VolumeMounts.extract(mappings, "/dir-one/test/dir")); + Set.of("pv-claim-one:/test-dir-1/"), VolumeMounts.extract(mappings, "/dir-one/test/dir")); assertEquals( - List.of("pv-claim-two:/test-dir-2/"), VolumeMounts.extract(mappings, "/dir-two/test/dir")); + Set.of("pv-claim-two:/test-dir-2/"), VolumeMounts.extract(mappings, "/dir-two/test/dir")); assertEquals( - List.of("pv-claim-three:/test-dir-3/"), + Set.of("pv-claim-three:/test-dir-3/"), VolumeMounts.extract(mappings, "/dir-three/test/dir")); } @Test public void multiMatchTest() { val mappings = - List.of( + Set.of( "pv-claim-one:/test-dir-1/,/vol-dir/", "pv-claim-one:/test-dir-2/,/vol-dir/", "pv-claim-one:/test-dir-3/,/vol-dir/"); assertEquals( - List.of( + Set.of( "pv-claim-one:/test-dir-1/", "pv-claim-one:/test-dir-2/", "pv-claim-one:/test-dir-3/"), VolumeMounts.extract(mappings, "/vol-dir/sub/dir")); } @Test - public void testExtractFromPropertiesWhenPresent() { - val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); - k8sProperties.setVolMountMappings(List.of("pv-claim:/some/dir,/some/dir/")); + public void testExtractFromPropertiesAndEngineParams() { + val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); + + // include some inconsistent matching strings (with and without /) + k8sProperties.setVolMountMappings( + Set.of( + "pv-claim-one:/test-dir-1/,/dir-one/", + "pv-claim-two:/test-dir-2/,/dir-two", + "pv-claim-three:/test-dir-3/,dir-three/")); + + val workflowEngineParams = + WorkflowEngineParams.builder() + .launchDir("/dir-one/test/dir") + .projectDir("/dir-two/test/dir") + .workDir("/dir-three/test/dir") + .build(); assertEquals( - List.of("pv-claim:/some/dir"), - VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); + Set.of( + "pv-claim-one:/test-dir-1/", + "pv-claim-two:/test-dir-2/", + "pv-claim-three:/test-dir-3/"), + VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @Test public void testExtractFromPropertiesWhenMissingVolMountsMapping() { - val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); - k8sProperties.setVolMounts(List.of("pv-claim:/some/dir")); + val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); + k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); + + val workflowEngineParams = + WorkflowEngineParams.builder() + .launchDir("/some/dir/launch/dir") + .projectDir("/some/dir/project/dir") + .workDir("/some/dir/work/dir") + .build(); assertEquals( - List.of("pv-claim:/some/dir"), - VolumeMounts.extract(k8sProperties, "/some/dir/with/sub/dir")); + Set.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @Test - public void testExtractFromPropertiesWhenMissingWorkdir() { - val k8sProperties = new NextflowProperties.K8sProperties(List.of("pv-claim:/some/dir")); - k8sProperties.setVolMountMappings(List.of("pv-claim:/some/dir,/some/dir/")); - k8sProperties.setVolMounts(List.of("pv-claim:/some/dir")); + public void testExtractFromPropertiesWhenMissingDirectories() { + val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); + k8sProperties.setVolMountMappings(Set.of("pv-claim:/some/dir,/some/dir/")); + k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); + + val workflowEngineParams = WorkflowEngineParams.builder().build(); - assertEquals(List.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, null)); + assertEquals( + Set.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); + } + + @Test + public void testExtractFromPropertiesDefaultVolMountsOnNoMatch() { + val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); + k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); + + val workflowEngineParams = + WorkflowEngineParams.builder() + .launchDir("/different/dir/launch/dir") + .projectDir("/different/dir/project/dir") + .workDir("/different/dir/work/dir") + .build(); + + assertEquals( + Set.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } } From d4e6f2dbc301b225a956f146d39d3be97fa08c1b Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 23 Aug 2021 16:41:37 -0400 Subject: [PATCH 05/26] applied "Fixes generated sources output dir" as per Workflow API PR --- pom.xml | 2 +- .../streams/schema/EngineParams.java | 983 --------------- .../streams/schema/RunState.java | 32 - .../streams/schema/WfMgmtRunMsg.java | 1115 ----------------- 4 files changed, 1 insertion(+), 2131 deletions(-) delete mode 100644 src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java delete mode 100644 src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java delete mode 100644 src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java diff --git a/pom.xml b/pom.xml index 8c97f80..8b8251c 100644 --- a/pom.xml +++ b/pom.xml @@ -185,7 +185,7 @@ ${project.basedir}/src/main/resources/avro/ - ${project.basedir}/src/main/java/ + ${project.build.directory}/generated-sources/avro String diff --git a/src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java b/src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java deleted file mode 100644 index 7e33b46..0000000 --- a/src/main/java/org/icgc/argo/workflow_management/streams/schema/EngineParams.java +++ /dev/null @@ -1,983 +0,0 @@ -/** - * Autogenerated by Avro - * - *

DO NOT EDIT DIRECTLY - */ -package org.icgc.argo.workflow_management.streams.schema; - -import org.apache.avro.message.BinaryMessageDecoder; -import org.apache.avro.message.BinaryMessageEncoder; -import org.apache.avro.message.SchemaStore; -import org.apache.avro.specific.SpecificData; - -@org.apache.avro.specific.AvroGenerated -public class EngineParams extends org.apache.avro.specific.SpecificRecordBase - implements org.apache.avro.specific.SpecificRecord { - private static final long serialVersionUID = 4292292046586660533L; - public static final org.apache.avro.Schema SCHEMA$ = - new org.apache.avro.Schema.Parser() - .parse( - "{\"type\":\"record\",\"name\":\"EngineParams\",\"namespace\":\"org.icgc.argo.workflow_management.streams.schema\",\"fields\":[{\"name\":\"defaultContainer\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"revision\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"launchDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"projectDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"latest\",\"type\":[\"null\",\"boolean\"],\"default\":null},{\"name\":\"resume\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null}]}"); - - public static org.apache.avro.Schema getClassSchema() { - return SCHEMA$; - } - - private static SpecificData MODEL$ = new SpecificData(); - - private static final BinaryMessageEncoder ENCODER = - new BinaryMessageEncoder(MODEL$, SCHEMA$); - - private static final BinaryMessageDecoder DECODER = - new BinaryMessageDecoder(MODEL$, SCHEMA$); - - /** - * Return the BinaryMessageEncoder instance used by this class. - * - * @return the message encoder used by this class - */ - public static BinaryMessageEncoder getEncoder() { - return ENCODER; - } - - /** - * Return the BinaryMessageDecoder instance used by this class. - * - * @return the message decoder used by this class - */ - public static BinaryMessageDecoder getDecoder() { - return DECODER; - } - - /** - * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link - * SchemaStore}. - * - * @param resolver a {@link SchemaStore} used to find schemas by fingerprint - * @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore - */ - public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { - return new BinaryMessageDecoder(MODEL$, SCHEMA$, resolver); - } - - /** - * Serializes this EngineParams to a ByteBuffer. - * - * @return a buffer holding the serialized data for this instance - * @throws java.io.IOException if this instance could not be serialized - */ - public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { - return ENCODER.encode(this); - } - - /** - * Deserializes a EngineParams from a ByteBuffer. - * - * @param b a byte buffer holding serialized data for an instance of this class - * @return a EngineParams instance decoded from the given buffer - * @throws java.io.IOException if the given bytes could not be deserialized into an instance of - * this class - */ - public static EngineParams fromByteBuffer(java.nio.ByteBuffer b) throws java.io.IOException { - return DECODER.decode(b); - } - - private java.lang.String defaultContainer; - private java.lang.String revision; - private java.lang.String launchDir; - private java.lang.String projectDir; - private java.lang.String workDir; - private java.lang.Boolean latest; - private java.lang.String resume; - - /** - * Default constructor. Note that this does not initialize fields to their default values from the - * schema. If that is desired then one should use newBuilder(). - */ - public EngineParams() {} - - /** - * All-args constructor. - * - * @param defaultContainer The new value for defaultContainer - * @param revision The new value for revision - * @param launchDir The new value for launchDir - * @param projectDir The new value for projectDir - * @param workDir The new value for workDir - * @param latest The new value for latest - * @param resume The new value for resume - */ - public EngineParams( - java.lang.String defaultContainer, - java.lang.String revision, - java.lang.String launchDir, - java.lang.String projectDir, - java.lang.String workDir, - java.lang.Boolean latest, - java.lang.String resume) { - this.defaultContainer = defaultContainer; - this.revision = revision; - this.launchDir = launchDir; - this.projectDir = projectDir; - this.workDir = workDir; - this.latest = latest; - this.resume = resume; - } - - public org.apache.avro.specific.SpecificData getSpecificData() { - return MODEL$; - } - - public org.apache.avro.Schema getSchema() { - return SCHEMA$; - } - // Used by DatumWriter. Applications should not call. - public java.lang.Object get(int field$) { - switch (field$) { - case 0: - return defaultContainer; - case 1: - return revision; - case 2: - return launchDir; - case 3: - return projectDir; - case 4: - return workDir; - case 5: - return latest; - case 6: - return resume; - default: - throw new IndexOutOfBoundsException("Invalid index: " + field$); - } - } - - // Used by DatumReader. Applications should not call. - @SuppressWarnings(value = "unchecked") - public void put(int field$, java.lang.Object value$) { - switch (field$) { - case 0: - defaultContainer = value$ != null ? value$.toString() : null; - break; - case 1: - revision = value$ != null ? value$.toString() : null; - break; - case 2: - launchDir = value$ != null ? value$.toString() : null; - break; - case 3: - projectDir = value$ != null ? value$.toString() : null; - break; - case 4: - workDir = value$ != null ? value$.toString() : null; - break; - case 5: - latest = (java.lang.Boolean) value$; - break; - case 6: - resume = value$ != null ? value$.toString() : null; - break; - default: - throw new IndexOutOfBoundsException("Invalid index: " + field$); - } - } - - /** - * Gets the value of the 'defaultContainer' field. - * - * @return The value of the 'defaultContainer' field. - */ - public java.lang.String getDefaultContainer() { - return defaultContainer; - } - - /** - * Sets the value of the 'defaultContainer' field. - * - * @param value the value to set. - */ - public void setDefaultContainer(java.lang.String value) { - this.defaultContainer = value; - } - - /** - * Gets the value of the 'revision' field. - * - * @return The value of the 'revision' field. - */ - public java.lang.String getRevision() { - return revision; - } - - /** - * Sets the value of the 'revision' field. - * - * @param value the value to set. - */ - public void setRevision(java.lang.String value) { - this.revision = value; - } - - /** - * Gets the value of the 'launchDir' field. - * - * @return The value of the 'launchDir' field. - */ - public java.lang.String getLaunchDir() { - return launchDir; - } - - /** - * Sets the value of the 'launchDir' field. - * - * @param value the value to set. - */ - public void setLaunchDir(java.lang.String value) { - this.launchDir = value; - } - - /** - * Gets the value of the 'projectDir' field. - * - * @return The value of the 'projectDir' field. - */ - public java.lang.String getProjectDir() { - return projectDir; - } - - /** - * Sets the value of the 'projectDir' field. - * - * @param value the value to set. - */ - public void setProjectDir(java.lang.String value) { - this.projectDir = value; - } - - /** - * Gets the value of the 'workDir' field. - * - * @return The value of the 'workDir' field. - */ - public java.lang.String getWorkDir() { - return workDir; - } - - /** - * Sets the value of the 'workDir' field. - * - * @param value the value to set. - */ - public void setWorkDir(java.lang.String value) { - this.workDir = value; - } - - /** - * Gets the value of the 'latest' field. - * - * @return The value of the 'latest' field. - */ - public java.lang.Boolean getLatest() { - return latest; - } - - /** - * Sets the value of the 'latest' field. - * - * @param value the value to set. - */ - public void setLatest(java.lang.Boolean value) { - this.latest = value; - } - - /** - * Gets the value of the 'resume' field. - * - * @return The value of the 'resume' field. - */ - public java.lang.String getResume() { - return resume; - } - - /** - * Sets the value of the 'resume' field. - * - * @param value the value to set. - */ - public void setResume(java.lang.String value) { - this.resume = value; - } - - /** - * Creates a new EngineParams RecordBuilder. - * - * @return A new EngineParams RecordBuilder - */ - public static org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder newBuilder() { - return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(); - } - - /** - * Creates a new EngineParams RecordBuilder by copying an existing Builder. - * - * @param other The existing builder to copy. - * @return A new EngineParams RecordBuilder - */ - public static org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder newBuilder( - org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder other) { - if (other == null) { - return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(); - } else { - return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(other); - } - } - - /** - * Creates a new EngineParams RecordBuilder by copying an existing EngineParams instance. - * - * @param other The existing instance to copy. - * @return A new EngineParams RecordBuilder - */ - public static org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder newBuilder( - org.icgc.argo.workflow_management.streams.schema.EngineParams other) { - if (other == null) { - return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(); - } else { - return new org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder(other); - } - } - - /** RecordBuilder for EngineParams instances. */ - @org.apache.avro.specific.AvroGenerated - public static class Builder - extends org.apache.avro.specific.SpecificRecordBuilderBase - implements org.apache.avro.data.RecordBuilder { - - private java.lang.String defaultContainer; - private java.lang.String revision; - private java.lang.String launchDir; - private java.lang.String projectDir; - private java.lang.String workDir; - private java.lang.Boolean latest; - private java.lang.String resume; - - /** Creates a new Builder */ - private Builder() { - super(SCHEMA$); - } - - /** - * Creates a Builder by copying an existing Builder. - * - * @param other The existing Builder to copy. - */ - private Builder(org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder other) { - super(other); - if (isValidValue(fields()[0], other.defaultContainer)) { - this.defaultContainer = data().deepCopy(fields()[0].schema(), other.defaultContainer); - fieldSetFlags()[0] = other.fieldSetFlags()[0]; - } - if (isValidValue(fields()[1], other.revision)) { - this.revision = data().deepCopy(fields()[1].schema(), other.revision); - fieldSetFlags()[1] = other.fieldSetFlags()[1]; - } - if (isValidValue(fields()[2], other.launchDir)) { - this.launchDir = data().deepCopy(fields()[2].schema(), other.launchDir); - fieldSetFlags()[2] = other.fieldSetFlags()[2]; - } - if (isValidValue(fields()[3], other.projectDir)) { - this.projectDir = data().deepCopy(fields()[3].schema(), other.projectDir); - fieldSetFlags()[3] = other.fieldSetFlags()[3]; - } - if (isValidValue(fields()[4], other.workDir)) { - this.workDir = data().deepCopy(fields()[4].schema(), other.workDir); - fieldSetFlags()[4] = other.fieldSetFlags()[4]; - } - if (isValidValue(fields()[5], other.latest)) { - this.latest = data().deepCopy(fields()[5].schema(), other.latest); - fieldSetFlags()[5] = other.fieldSetFlags()[5]; - } - if (isValidValue(fields()[6], other.resume)) { - this.resume = data().deepCopy(fields()[6].schema(), other.resume); - fieldSetFlags()[6] = other.fieldSetFlags()[6]; - } - } - - /** - * Creates a Builder by copying an existing EngineParams instance - * - * @param other The existing instance to copy. - */ - private Builder(org.icgc.argo.workflow_management.streams.schema.EngineParams other) { - super(SCHEMA$); - if (isValidValue(fields()[0], other.defaultContainer)) { - this.defaultContainer = data().deepCopy(fields()[0].schema(), other.defaultContainer); - fieldSetFlags()[0] = true; - } - if (isValidValue(fields()[1], other.revision)) { - this.revision = data().deepCopy(fields()[1].schema(), other.revision); - fieldSetFlags()[1] = true; - } - if (isValidValue(fields()[2], other.launchDir)) { - this.launchDir = data().deepCopy(fields()[2].schema(), other.launchDir); - fieldSetFlags()[2] = true; - } - if (isValidValue(fields()[3], other.projectDir)) { - this.projectDir = data().deepCopy(fields()[3].schema(), other.projectDir); - fieldSetFlags()[3] = true; - } - if (isValidValue(fields()[4], other.workDir)) { - this.workDir = data().deepCopy(fields()[4].schema(), other.workDir); - fieldSetFlags()[4] = true; - } - if (isValidValue(fields()[5], other.latest)) { - this.latest = data().deepCopy(fields()[5].schema(), other.latest); - fieldSetFlags()[5] = true; - } - if (isValidValue(fields()[6], other.resume)) { - this.resume = data().deepCopy(fields()[6].schema(), other.resume); - fieldSetFlags()[6] = true; - } - } - - /** - * Gets the value of the 'defaultContainer' field. - * - * @return The value. - */ - public java.lang.String getDefaultContainer() { - return defaultContainer; - } - - /** - * Sets the value of the 'defaultContainer' field. - * - * @param value The value of 'defaultContainer'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder - setDefaultContainer(java.lang.String value) { - validate(fields()[0], value); - this.defaultContainer = value; - fieldSetFlags()[0] = true; - return this; - } - - /** - * Checks whether the 'defaultContainer' field has been set. - * - * @return True if the 'defaultContainer' field has been set, false otherwise. - */ - public boolean hasDefaultContainer() { - return fieldSetFlags()[0]; - } - - /** - * Clears the value of the 'defaultContainer' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder - clearDefaultContainer() { - defaultContainer = null; - fieldSetFlags()[0] = false; - return this; - } - - /** - * Gets the value of the 'revision' field. - * - * @return The value. - */ - public java.lang.String getRevision() { - return revision; - } - - /** - * Sets the value of the 'revision' field. - * - * @param value The value of 'revision'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setRevision( - java.lang.String value) { - validate(fields()[1], value); - this.revision = value; - fieldSetFlags()[1] = true; - return this; - } - - /** - * Checks whether the 'revision' field has been set. - * - * @return True if the 'revision' field has been set, false otherwise. - */ - public boolean hasRevision() { - return fieldSetFlags()[1]; - } - - /** - * Clears the value of the 'revision' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearRevision() { - revision = null; - fieldSetFlags()[1] = false; - return this; - } - - /** - * Gets the value of the 'launchDir' field. - * - * @return The value. - */ - public java.lang.String getLaunchDir() { - return launchDir; - } - - /** - * Sets the value of the 'launchDir' field. - * - * @param value The value of 'launchDir'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setLaunchDir( - java.lang.String value) { - validate(fields()[2], value); - this.launchDir = value; - fieldSetFlags()[2] = true; - return this; - } - - /** - * Checks whether the 'launchDir' field has been set. - * - * @return True if the 'launchDir' field has been set, false otherwise. - */ - public boolean hasLaunchDir() { - return fieldSetFlags()[2]; - } - - /** - * Clears the value of the 'launchDir' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearLaunchDir() { - launchDir = null; - fieldSetFlags()[2] = false; - return this; - } - - /** - * Gets the value of the 'projectDir' field. - * - * @return The value. - */ - public java.lang.String getProjectDir() { - return projectDir; - } - - /** - * Sets the value of the 'projectDir' field. - * - * @param value The value of 'projectDir'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setProjectDir( - java.lang.String value) { - validate(fields()[3], value); - this.projectDir = value; - fieldSetFlags()[3] = true; - return this; - } - - /** - * Checks whether the 'projectDir' field has been set. - * - * @return True if the 'projectDir' field has been set, false otherwise. - */ - public boolean hasProjectDir() { - return fieldSetFlags()[3]; - } - - /** - * Clears the value of the 'projectDir' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearProjectDir() { - projectDir = null; - fieldSetFlags()[3] = false; - return this; - } - - /** - * Gets the value of the 'workDir' field. - * - * @return The value. - */ - public java.lang.String getWorkDir() { - return workDir; - } - - /** - * Sets the value of the 'workDir' field. - * - * @param value The value of 'workDir'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setWorkDir( - java.lang.String value) { - validate(fields()[4], value); - this.workDir = value; - fieldSetFlags()[4] = true; - return this; - } - - /** - * Checks whether the 'workDir' field has been set. - * - * @return True if the 'workDir' field has been set, false otherwise. - */ - public boolean hasWorkDir() { - return fieldSetFlags()[4]; - } - - /** - * Clears the value of the 'workDir' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearWorkDir() { - workDir = null; - fieldSetFlags()[4] = false; - return this; - } - - /** - * Gets the value of the 'latest' field. - * - * @return The value. - */ - public java.lang.Boolean getLatest() { - return latest; - } - - /** - * Sets the value of the 'latest' field. - * - * @param value The value of 'latest'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setLatest( - java.lang.Boolean value) { - validate(fields()[5], value); - this.latest = value; - fieldSetFlags()[5] = true; - return this; - } - - /** - * Checks whether the 'latest' field has been set. - * - * @return True if the 'latest' field has been set, false otherwise. - */ - public boolean hasLatest() { - return fieldSetFlags()[5]; - } - - /** - * Clears the value of the 'latest' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearLatest() { - latest = null; - fieldSetFlags()[5] = false; - return this; - } - - /** - * Gets the value of the 'resume' field. - * - * @return The value. - */ - public java.lang.String getResume() { - return resume; - } - - /** - * Sets the value of the 'resume' field. - * - * @param value The value of 'resume'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder setResume( - java.lang.String value) { - validate(fields()[6], value); - this.resume = value; - fieldSetFlags()[6] = true; - return this; - } - - /** - * Checks whether the 'resume' field has been set. - * - * @return True if the 'resume' field has been set, false otherwise. - */ - public boolean hasResume() { - return fieldSetFlags()[6]; - } - - /** - * Clears the value of the 'resume' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder clearResume() { - resume = null; - fieldSetFlags()[6] = false; - return this; - } - - @Override - @SuppressWarnings("unchecked") - public EngineParams build() { - try { - EngineParams record = new EngineParams(); - record.defaultContainer = - fieldSetFlags()[0] - ? this.defaultContainer - : (java.lang.String) defaultValue(fields()[0]); - record.revision = - fieldSetFlags()[1] ? this.revision : (java.lang.String) defaultValue(fields()[1]); - record.launchDir = - fieldSetFlags()[2] ? this.launchDir : (java.lang.String) defaultValue(fields()[2]); - record.projectDir = - fieldSetFlags()[3] ? this.projectDir : (java.lang.String) defaultValue(fields()[3]); - record.workDir = - fieldSetFlags()[4] ? this.workDir : (java.lang.String) defaultValue(fields()[4]); - record.latest = - fieldSetFlags()[5] ? this.latest : (java.lang.Boolean) defaultValue(fields()[5]); - record.resume = - fieldSetFlags()[6] ? this.resume : (java.lang.String) defaultValue(fields()[6]); - return record; - } catch (org.apache.avro.AvroMissingFieldException e) { - throw e; - } catch (java.lang.Exception e) { - throw new org.apache.avro.AvroRuntimeException(e); - } - } - } - - @SuppressWarnings("unchecked") - private static final org.apache.avro.io.DatumWriter WRITER$ = - (org.apache.avro.io.DatumWriter) MODEL$.createDatumWriter(SCHEMA$); - - @Override - public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException { - WRITER$.write(this, SpecificData.getEncoder(out)); - } - - @SuppressWarnings("unchecked") - private static final org.apache.avro.io.DatumReader READER$ = - (org.apache.avro.io.DatumReader) MODEL$.createDatumReader(SCHEMA$); - - @Override - public void readExternal(java.io.ObjectInput in) throws java.io.IOException { - READER$.read(this, SpecificData.getDecoder(in)); - } - - @Override - protected boolean hasCustomCoders() { - return true; - } - - @Override - public void customEncode(org.apache.avro.io.Encoder out) throws java.io.IOException { - if (this.defaultContainer == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.defaultContainer); - } - - if (this.revision == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.revision); - } - - if (this.launchDir == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.launchDir); - } - - if (this.projectDir == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.projectDir); - } - - if (this.workDir == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.workDir); - } - - if (this.latest == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeBoolean(this.latest); - } - - if (this.resume == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.resume); - } - } - - @Override - public void customDecode(org.apache.avro.io.ResolvingDecoder in) throws java.io.IOException { - org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff(); - if (fieldOrder == null) { - if (in.readIndex() != 1) { - in.readNull(); - this.defaultContainer = null; - } else { - this.defaultContainer = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.revision = null; - } else { - this.revision = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.launchDir = null; - } else { - this.launchDir = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.projectDir = null; - } else { - this.projectDir = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.workDir = null; - } else { - this.workDir = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.latest = null; - } else { - this.latest = in.readBoolean(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.resume = null; - } else { - this.resume = in.readString(); - } - - } else { - for (int i = 0; i < 7; i++) { - switch (fieldOrder[i].pos()) { - case 0: - if (in.readIndex() != 1) { - in.readNull(); - this.defaultContainer = null; - } else { - this.defaultContainer = in.readString(); - } - break; - - case 1: - if (in.readIndex() != 1) { - in.readNull(); - this.revision = null; - } else { - this.revision = in.readString(); - } - break; - - case 2: - if (in.readIndex() != 1) { - in.readNull(); - this.launchDir = null; - } else { - this.launchDir = in.readString(); - } - break; - - case 3: - if (in.readIndex() != 1) { - in.readNull(); - this.projectDir = null; - } else { - this.projectDir = in.readString(); - } - break; - - case 4: - if (in.readIndex() != 1) { - in.readNull(); - this.workDir = null; - } else { - this.workDir = in.readString(); - } - break; - - case 5: - if (in.readIndex() != 1) { - in.readNull(); - this.latest = null; - } else { - this.latest = in.readBoolean(); - } - break; - - case 6: - if (in.readIndex() != 1) { - in.readNull(); - this.resume = null; - } else { - this.resume = in.readString(); - } - break; - - default: - throw new java.io.IOException("Corrupt ResolvingDecoder."); - } - } - } - } -} diff --git a/src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java b/src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java deleted file mode 100644 index ce47619..0000000 --- a/src/main/java/org/icgc/argo/workflow_management/streams/schema/RunState.java +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Autogenerated by Avro - * - *

DO NOT EDIT DIRECTLY - */ -package org.icgc.argo.workflow_management.streams.schema; - -@org.apache.avro.specific.AvroGenerated -public enum RunState implements org.apache.avro.generic.GenericEnumSymbol { - UNKNOWN, - QUEUED, - INITIALIZING, - RUNNING, - PAUSED, - CANCELING, - CANCELED, - COMPLETE, - EXECUTOR_ERROR, - SYSTEM_ERROR; - public static final org.apache.avro.Schema SCHEMA$ = - new org.apache.avro.Schema.Parser() - .parse( - "{\"type\":\"enum\",\"name\":\"RunState\",\"namespace\":\"org.icgc.argo.workflow_management.streams.schema\",\"symbols\":[\"UNKNOWN\",\"QUEUED\",\"INITIALIZING\",\"RUNNING\",\"PAUSED\",\"CANCELING\",\"CANCELED\",\"COMPLETE\",\"EXECUTOR_ERROR\",\"SYSTEM_ERROR\"]}"); - - public static org.apache.avro.Schema getClassSchema() { - return SCHEMA$; - } - - public org.apache.avro.Schema getSchema() { - return SCHEMA$; - } -} diff --git a/src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java b/src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java deleted file mode 100644 index 305efd4..0000000 --- a/src/main/java/org/icgc/argo/workflow_management/streams/schema/WfMgmtRunMsg.java +++ /dev/null @@ -1,1115 +0,0 @@ -/** - * Autogenerated by Avro - * - *

DO NOT EDIT DIRECTLY - */ -package org.icgc.argo.workflow_management.streams.schema; - -import org.apache.avro.message.BinaryMessageDecoder; -import org.apache.avro.message.BinaryMessageEncoder; -import org.apache.avro.message.SchemaStore; -import org.apache.avro.specific.SpecificData; - -@org.apache.avro.specific.AvroGenerated -public class WfMgmtRunMsg extends org.apache.avro.specific.SpecificRecordBase - implements org.apache.avro.specific.SpecificRecord { - private static final long serialVersionUID = -2804506531583278262L; - public static final org.apache.avro.Schema SCHEMA$ = - new org.apache.avro.Schema.Parser() - .parse( - "{\"type\":\"record\",\"name\":\"WfMgmtRunMsg\",\"namespace\":\"org.icgc.argo.workflow_management.streams.schema\",\"fields\":[{\"name\":\"timestamp\",\"type\":\"long\",\"logicalType\":\"date\"},{\"name\":\"runId\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"state\",\"type\":{\"type\":\"enum\",\"name\":\"RunState\",\"symbols\":[\"UNKNOWN\",\"QUEUED\",\"INITIALIZING\",\"RUNNING\",\"PAUSED\",\"CANCELING\",\"CANCELED\",\"COMPLETE\",\"EXECUTOR_ERROR\",\"SYSTEM_ERROR\"]}},{\"name\":\"workflowUrl\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowType\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowTypeVersion\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowParamsJsonStr\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workflowEngineParams\",\"type\":{\"type\":\"record\",\"name\":\"EngineParams\",\"fields\":[{\"name\":\"defaultContainer\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"revision\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"launchDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"projectDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"workDir\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null},{\"name\":\"latest\",\"type\":[\"null\",\"boolean\"],\"default\":null},{\"name\":\"resume\",\"type\":[\"null\",{\"type\":\"string\",\"avro.java.string\":\"String\"}],\"default\":null}]}}]}"); - - public static org.apache.avro.Schema getClassSchema() { - return SCHEMA$; - } - - private static SpecificData MODEL$ = new SpecificData(); - - private static final BinaryMessageEncoder ENCODER = - new BinaryMessageEncoder(MODEL$, SCHEMA$); - - private static final BinaryMessageDecoder DECODER = - new BinaryMessageDecoder(MODEL$, SCHEMA$); - - /** - * Return the BinaryMessageEncoder instance used by this class. - * - * @return the message encoder used by this class - */ - public static BinaryMessageEncoder getEncoder() { - return ENCODER; - } - - /** - * Return the BinaryMessageDecoder instance used by this class. - * - * @return the message decoder used by this class - */ - public static BinaryMessageDecoder getDecoder() { - return DECODER; - } - - /** - * Create a new BinaryMessageDecoder instance for this class that uses the specified {@link - * SchemaStore}. - * - * @param resolver a {@link SchemaStore} used to find schemas by fingerprint - * @return a BinaryMessageDecoder instance for this class backed by the given SchemaStore - */ - public static BinaryMessageDecoder createDecoder(SchemaStore resolver) { - return new BinaryMessageDecoder(MODEL$, SCHEMA$, resolver); - } - - /** - * Serializes this WfMgmtRunMsg to a ByteBuffer. - * - * @return a buffer holding the serialized data for this instance - * @throws java.io.IOException if this instance could not be serialized - */ - public java.nio.ByteBuffer toByteBuffer() throws java.io.IOException { - return ENCODER.encode(this); - } - - /** - * Deserializes a WfMgmtRunMsg from a ByteBuffer. - * - * @param b a byte buffer holding serialized data for an instance of this class - * @return a WfMgmtRunMsg instance decoded from the given buffer - * @throws java.io.IOException if the given bytes could not be deserialized into an instance of - * this class - */ - public static WfMgmtRunMsg fromByteBuffer(java.nio.ByteBuffer b) throws java.io.IOException { - return DECODER.decode(b); - } - - private long timestamp; - private java.lang.String runId; - private org.icgc.argo.workflow_management.streams.schema.RunState state; - private java.lang.String workflowUrl; - private java.lang.String workflowType; - private java.lang.String workflowTypeVersion; - private java.lang.String workflowParamsJsonStr; - private org.icgc.argo.workflow_management.streams.schema.EngineParams workflowEngineParams; - - /** - * Default constructor. Note that this does not initialize fields to their default values from the - * schema. If that is desired then one should use newBuilder(). - */ - public WfMgmtRunMsg() {} - - /** - * All-args constructor. - * - * @param timestamp The new value for timestamp - * @param runId The new value for runId - * @param state The new value for state - * @param workflowUrl The new value for workflowUrl - * @param workflowType The new value for workflowType - * @param workflowTypeVersion The new value for workflowTypeVersion - * @param workflowParamsJsonStr The new value for workflowParamsJsonStr - * @param workflowEngineParams The new value for workflowEngineParams - */ - public WfMgmtRunMsg( - java.lang.Long timestamp, - java.lang.String runId, - org.icgc.argo.workflow_management.streams.schema.RunState state, - java.lang.String workflowUrl, - java.lang.String workflowType, - java.lang.String workflowTypeVersion, - java.lang.String workflowParamsJsonStr, - org.icgc.argo.workflow_management.streams.schema.EngineParams workflowEngineParams) { - this.timestamp = timestamp; - this.runId = runId; - this.state = state; - this.workflowUrl = workflowUrl; - this.workflowType = workflowType; - this.workflowTypeVersion = workflowTypeVersion; - this.workflowParamsJsonStr = workflowParamsJsonStr; - this.workflowEngineParams = workflowEngineParams; - } - - public org.apache.avro.specific.SpecificData getSpecificData() { - return MODEL$; - } - - public org.apache.avro.Schema getSchema() { - return SCHEMA$; - } - // Used by DatumWriter. Applications should not call. - public java.lang.Object get(int field$) { - switch (field$) { - case 0: - return timestamp; - case 1: - return runId; - case 2: - return state; - case 3: - return workflowUrl; - case 4: - return workflowType; - case 5: - return workflowTypeVersion; - case 6: - return workflowParamsJsonStr; - case 7: - return workflowEngineParams; - default: - throw new IndexOutOfBoundsException("Invalid index: " + field$); - } - } - - // Used by DatumReader. Applications should not call. - @SuppressWarnings(value = "unchecked") - public void put(int field$, java.lang.Object value$) { - switch (field$) { - case 0: - timestamp = (java.lang.Long) value$; - break; - case 1: - runId = value$ != null ? value$.toString() : null; - break; - case 2: - state = (org.icgc.argo.workflow_management.streams.schema.RunState) value$; - break; - case 3: - workflowUrl = value$ != null ? value$.toString() : null; - break; - case 4: - workflowType = value$ != null ? value$.toString() : null; - break; - case 5: - workflowTypeVersion = value$ != null ? value$.toString() : null; - break; - case 6: - workflowParamsJsonStr = value$ != null ? value$.toString() : null; - break; - case 7: - workflowEngineParams = - (org.icgc.argo.workflow_management.streams.schema.EngineParams) value$; - break; - default: - throw new IndexOutOfBoundsException("Invalid index: " + field$); - } - } - - /** - * Gets the value of the 'timestamp' field. - * - * @return The value of the 'timestamp' field. - */ - public long getTimestamp() { - return timestamp; - } - - /** - * Sets the value of the 'timestamp' field. - * - * @param value the value to set. - */ - public void setTimestamp(long value) { - this.timestamp = value; - } - - /** - * Gets the value of the 'runId' field. - * - * @return The value of the 'runId' field. - */ - public java.lang.String getRunId() { - return runId; - } - - /** - * Sets the value of the 'runId' field. - * - * @param value the value to set. - */ - public void setRunId(java.lang.String value) { - this.runId = value; - } - - /** - * Gets the value of the 'state' field. - * - * @return The value of the 'state' field. - */ - public org.icgc.argo.workflow_management.streams.schema.RunState getState() { - return state; - } - - /** - * Sets the value of the 'state' field. - * - * @param value the value to set. - */ - public void setState(org.icgc.argo.workflow_management.streams.schema.RunState value) { - this.state = value; - } - - /** - * Gets the value of the 'workflowUrl' field. - * - * @return The value of the 'workflowUrl' field. - */ - public java.lang.String getWorkflowUrl() { - return workflowUrl; - } - - /** - * Sets the value of the 'workflowUrl' field. - * - * @param value the value to set. - */ - public void setWorkflowUrl(java.lang.String value) { - this.workflowUrl = value; - } - - /** - * Gets the value of the 'workflowType' field. - * - * @return The value of the 'workflowType' field. - */ - public java.lang.String getWorkflowType() { - return workflowType; - } - - /** - * Sets the value of the 'workflowType' field. - * - * @param value the value to set. - */ - public void setWorkflowType(java.lang.String value) { - this.workflowType = value; - } - - /** - * Gets the value of the 'workflowTypeVersion' field. - * - * @return The value of the 'workflowTypeVersion' field. - */ - public java.lang.String getWorkflowTypeVersion() { - return workflowTypeVersion; - } - - /** - * Sets the value of the 'workflowTypeVersion' field. - * - * @param value the value to set. - */ - public void setWorkflowTypeVersion(java.lang.String value) { - this.workflowTypeVersion = value; - } - - /** - * Gets the value of the 'workflowParamsJsonStr' field. - * - * @return The value of the 'workflowParamsJsonStr' field. - */ - public java.lang.String getWorkflowParamsJsonStr() { - return workflowParamsJsonStr; - } - - /** - * Sets the value of the 'workflowParamsJsonStr' field. - * - * @param value the value to set. - */ - public void setWorkflowParamsJsonStr(java.lang.String value) { - this.workflowParamsJsonStr = value; - } - - /** - * Gets the value of the 'workflowEngineParams' field. - * - * @return The value of the 'workflowEngineParams' field. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams getWorkflowEngineParams() { - return workflowEngineParams; - } - - /** - * Sets the value of the 'workflowEngineParams' field. - * - * @param value the value to set. - */ - public void setWorkflowEngineParams( - org.icgc.argo.workflow_management.streams.schema.EngineParams value) { - this.workflowEngineParams = value; - } - - /** - * Creates a new WfMgmtRunMsg RecordBuilder. - * - * @return A new WfMgmtRunMsg RecordBuilder - */ - public static org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder newBuilder() { - return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(); - } - - /** - * Creates a new WfMgmtRunMsg RecordBuilder by copying an existing Builder. - * - * @param other The existing builder to copy. - * @return A new WfMgmtRunMsg RecordBuilder - */ - public static org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder newBuilder( - org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder other) { - if (other == null) { - return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(); - } else { - return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(other); - } - } - - /** - * Creates a new WfMgmtRunMsg RecordBuilder by copying an existing WfMgmtRunMsg instance. - * - * @param other The existing instance to copy. - * @return A new WfMgmtRunMsg RecordBuilder - */ - public static org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder newBuilder( - org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg other) { - if (other == null) { - return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(); - } else { - return new org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder(other); - } - } - - /** RecordBuilder for WfMgmtRunMsg instances. */ - @org.apache.avro.specific.AvroGenerated - public static class Builder - extends org.apache.avro.specific.SpecificRecordBuilderBase - implements org.apache.avro.data.RecordBuilder { - - private long timestamp; - private java.lang.String runId; - private org.icgc.argo.workflow_management.streams.schema.RunState state; - private java.lang.String workflowUrl; - private java.lang.String workflowType; - private java.lang.String workflowTypeVersion; - private java.lang.String workflowParamsJsonStr; - private org.icgc.argo.workflow_management.streams.schema.EngineParams workflowEngineParams; - private org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder - workflowEngineParamsBuilder; - - /** Creates a new Builder */ - private Builder() { - super(SCHEMA$); - } - - /** - * Creates a Builder by copying an existing Builder. - * - * @param other The existing Builder to copy. - */ - private Builder(org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder other) { - super(other); - if (isValidValue(fields()[0], other.timestamp)) { - this.timestamp = data().deepCopy(fields()[0].schema(), other.timestamp); - fieldSetFlags()[0] = other.fieldSetFlags()[0]; - } - if (isValidValue(fields()[1], other.runId)) { - this.runId = data().deepCopy(fields()[1].schema(), other.runId); - fieldSetFlags()[1] = other.fieldSetFlags()[1]; - } - if (isValidValue(fields()[2], other.state)) { - this.state = data().deepCopy(fields()[2].schema(), other.state); - fieldSetFlags()[2] = other.fieldSetFlags()[2]; - } - if (isValidValue(fields()[3], other.workflowUrl)) { - this.workflowUrl = data().deepCopy(fields()[3].schema(), other.workflowUrl); - fieldSetFlags()[3] = other.fieldSetFlags()[3]; - } - if (isValidValue(fields()[4], other.workflowType)) { - this.workflowType = data().deepCopy(fields()[4].schema(), other.workflowType); - fieldSetFlags()[4] = other.fieldSetFlags()[4]; - } - if (isValidValue(fields()[5], other.workflowTypeVersion)) { - this.workflowTypeVersion = data().deepCopy(fields()[5].schema(), other.workflowTypeVersion); - fieldSetFlags()[5] = other.fieldSetFlags()[5]; - } - if (isValidValue(fields()[6], other.workflowParamsJsonStr)) { - this.workflowParamsJsonStr = - data().deepCopy(fields()[6].schema(), other.workflowParamsJsonStr); - fieldSetFlags()[6] = other.fieldSetFlags()[6]; - } - if (isValidValue(fields()[7], other.workflowEngineParams)) { - this.workflowEngineParams = - data().deepCopy(fields()[7].schema(), other.workflowEngineParams); - fieldSetFlags()[7] = other.fieldSetFlags()[7]; - } - if (other.hasWorkflowEngineParamsBuilder()) { - this.workflowEngineParamsBuilder = - org.icgc.argo.workflow_management.streams.schema.EngineParams.newBuilder( - other.getWorkflowEngineParamsBuilder()); - } - } - - /** - * Creates a Builder by copying an existing WfMgmtRunMsg instance - * - * @param other The existing instance to copy. - */ - private Builder(org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg other) { - super(SCHEMA$); - if (isValidValue(fields()[0], other.timestamp)) { - this.timestamp = data().deepCopy(fields()[0].schema(), other.timestamp); - fieldSetFlags()[0] = true; - } - if (isValidValue(fields()[1], other.runId)) { - this.runId = data().deepCopy(fields()[1].schema(), other.runId); - fieldSetFlags()[1] = true; - } - if (isValidValue(fields()[2], other.state)) { - this.state = data().deepCopy(fields()[2].schema(), other.state); - fieldSetFlags()[2] = true; - } - if (isValidValue(fields()[3], other.workflowUrl)) { - this.workflowUrl = data().deepCopy(fields()[3].schema(), other.workflowUrl); - fieldSetFlags()[3] = true; - } - if (isValidValue(fields()[4], other.workflowType)) { - this.workflowType = data().deepCopy(fields()[4].schema(), other.workflowType); - fieldSetFlags()[4] = true; - } - if (isValidValue(fields()[5], other.workflowTypeVersion)) { - this.workflowTypeVersion = data().deepCopy(fields()[5].schema(), other.workflowTypeVersion); - fieldSetFlags()[5] = true; - } - if (isValidValue(fields()[6], other.workflowParamsJsonStr)) { - this.workflowParamsJsonStr = - data().deepCopy(fields()[6].schema(), other.workflowParamsJsonStr); - fieldSetFlags()[6] = true; - } - if (isValidValue(fields()[7], other.workflowEngineParams)) { - this.workflowEngineParams = - data().deepCopy(fields()[7].schema(), other.workflowEngineParams); - fieldSetFlags()[7] = true; - } - this.workflowEngineParamsBuilder = null; - } - - /** - * Gets the value of the 'timestamp' field. - * - * @return The value. - */ - public long getTimestamp() { - return timestamp; - } - - /** - * Sets the value of the 'timestamp' field. - * - * @param value The value of 'timestamp'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setTimestamp( - long value) { - validate(fields()[0], value); - this.timestamp = value; - fieldSetFlags()[0] = true; - return this; - } - - /** - * Checks whether the 'timestamp' field has been set. - * - * @return True if the 'timestamp' field has been set, false otherwise. - */ - public boolean hasTimestamp() { - return fieldSetFlags()[0]; - } - - /** - * Clears the value of the 'timestamp' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder clearTimestamp() { - fieldSetFlags()[0] = false; - return this; - } - - /** - * Gets the value of the 'runId' field. - * - * @return The value. - */ - public java.lang.String getRunId() { - return runId; - } - - /** - * Sets the value of the 'runId' field. - * - * @param value The value of 'runId'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setRunId( - java.lang.String value) { - validate(fields()[1], value); - this.runId = value; - fieldSetFlags()[1] = true; - return this; - } - - /** - * Checks whether the 'runId' field has been set. - * - * @return True if the 'runId' field has been set, false otherwise. - */ - public boolean hasRunId() { - return fieldSetFlags()[1]; - } - - /** - * Clears the value of the 'runId' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder clearRunId() { - runId = null; - fieldSetFlags()[1] = false; - return this; - } - - /** - * Gets the value of the 'state' field. - * - * @return The value. - */ - public org.icgc.argo.workflow_management.streams.schema.RunState getState() { - return state; - } - - /** - * Sets the value of the 'state' field. - * - * @param value The value of 'state'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setState( - org.icgc.argo.workflow_management.streams.schema.RunState value) { - validate(fields()[2], value); - this.state = value; - fieldSetFlags()[2] = true; - return this; - } - - /** - * Checks whether the 'state' field has been set. - * - * @return True if the 'state' field has been set, false otherwise. - */ - public boolean hasState() { - return fieldSetFlags()[2]; - } - - /** - * Clears the value of the 'state' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder clearState() { - state = null; - fieldSetFlags()[2] = false; - return this; - } - - /** - * Gets the value of the 'workflowUrl' field. - * - * @return The value. - */ - public java.lang.String getWorkflowUrl() { - return workflowUrl; - } - - /** - * Sets the value of the 'workflowUrl' field. - * - * @param value The value of 'workflowUrl'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setWorkflowUrl( - java.lang.String value) { - validate(fields()[3], value); - this.workflowUrl = value; - fieldSetFlags()[3] = true; - return this; - } - - /** - * Checks whether the 'workflowUrl' field has been set. - * - * @return True if the 'workflowUrl' field has been set, false otherwise. - */ - public boolean hasWorkflowUrl() { - return fieldSetFlags()[3]; - } - - /** - * Clears the value of the 'workflowUrl' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - clearWorkflowUrl() { - workflowUrl = null; - fieldSetFlags()[3] = false; - return this; - } - - /** - * Gets the value of the 'workflowType' field. - * - * @return The value. - */ - public java.lang.String getWorkflowType() { - return workflowType; - } - - /** - * Sets the value of the 'workflowType' field. - * - * @param value The value of 'workflowType'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder setWorkflowType( - java.lang.String value) { - validate(fields()[4], value); - this.workflowType = value; - fieldSetFlags()[4] = true; - return this; - } - - /** - * Checks whether the 'workflowType' field has been set. - * - * @return True if the 'workflowType' field has been set, false otherwise. - */ - public boolean hasWorkflowType() { - return fieldSetFlags()[4]; - } - - /** - * Clears the value of the 'workflowType' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - clearWorkflowType() { - workflowType = null; - fieldSetFlags()[4] = false; - return this; - } - - /** - * Gets the value of the 'workflowTypeVersion' field. - * - * @return The value. - */ - public java.lang.String getWorkflowTypeVersion() { - return workflowTypeVersion; - } - - /** - * Sets the value of the 'workflowTypeVersion' field. - * - * @param value The value of 'workflowTypeVersion'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - setWorkflowTypeVersion(java.lang.String value) { - validate(fields()[5], value); - this.workflowTypeVersion = value; - fieldSetFlags()[5] = true; - return this; - } - - /** - * Checks whether the 'workflowTypeVersion' field has been set. - * - * @return True if the 'workflowTypeVersion' field has been set, false otherwise. - */ - public boolean hasWorkflowTypeVersion() { - return fieldSetFlags()[5]; - } - - /** - * Clears the value of the 'workflowTypeVersion' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - clearWorkflowTypeVersion() { - workflowTypeVersion = null; - fieldSetFlags()[5] = false; - return this; - } - - /** - * Gets the value of the 'workflowParamsJsonStr' field. - * - * @return The value. - */ - public java.lang.String getWorkflowParamsJsonStr() { - return workflowParamsJsonStr; - } - - /** - * Sets the value of the 'workflowParamsJsonStr' field. - * - * @param value The value of 'workflowParamsJsonStr'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - setWorkflowParamsJsonStr(java.lang.String value) { - validate(fields()[6], value); - this.workflowParamsJsonStr = value; - fieldSetFlags()[6] = true; - return this; - } - - /** - * Checks whether the 'workflowParamsJsonStr' field has been set. - * - * @return True if the 'workflowParamsJsonStr' field has been set, false otherwise. - */ - public boolean hasWorkflowParamsJsonStr() { - return fieldSetFlags()[6]; - } - - /** - * Clears the value of the 'workflowParamsJsonStr' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - clearWorkflowParamsJsonStr() { - workflowParamsJsonStr = null; - fieldSetFlags()[6] = false; - return this; - } - - /** - * Gets the value of the 'workflowEngineParams' field. - * - * @return The value. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams getWorkflowEngineParams() { - return workflowEngineParams; - } - - /** - * Sets the value of the 'workflowEngineParams' field. - * - * @param value The value of 'workflowEngineParams'. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - setWorkflowEngineParams( - org.icgc.argo.workflow_management.streams.schema.EngineParams value) { - validate(fields()[7], value); - this.workflowEngineParamsBuilder = null; - this.workflowEngineParams = value; - fieldSetFlags()[7] = true; - return this; - } - - /** - * Checks whether the 'workflowEngineParams' field has been set. - * - * @return True if the 'workflowEngineParams' field has been set, false otherwise. - */ - public boolean hasWorkflowEngineParams() { - return fieldSetFlags()[7]; - } - - /** - * Gets the Builder instance for the 'workflowEngineParams' field and creates one if it doesn't - * exist yet. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder - getWorkflowEngineParamsBuilder() { - if (workflowEngineParamsBuilder == null) { - if (hasWorkflowEngineParams()) { - setWorkflowEngineParamsBuilder( - org.icgc.argo.workflow_management.streams.schema.EngineParams.newBuilder( - workflowEngineParams)); - } else { - setWorkflowEngineParamsBuilder( - org.icgc.argo.workflow_management.streams.schema.EngineParams.newBuilder()); - } - } - return workflowEngineParamsBuilder; - } - - /** - * Sets the Builder instance for the 'workflowEngineParams' field - * - * @param value The builder instance that must be set. - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - setWorkflowEngineParamsBuilder( - org.icgc.argo.workflow_management.streams.schema.EngineParams.Builder value) { - clearWorkflowEngineParams(); - workflowEngineParamsBuilder = value; - return this; - } - - /** - * Checks whether the 'workflowEngineParams' field has an active Builder instance - * - * @return True if the 'workflowEngineParams' field has an active Builder instance - */ - public boolean hasWorkflowEngineParamsBuilder() { - return workflowEngineParamsBuilder != null; - } - - /** - * Clears the value of the 'workflowEngineParams' field. - * - * @return This builder. - */ - public org.icgc.argo.workflow_management.streams.schema.WfMgmtRunMsg.Builder - clearWorkflowEngineParams() { - workflowEngineParams = null; - workflowEngineParamsBuilder = null; - fieldSetFlags()[7] = false; - return this; - } - - @Override - @SuppressWarnings("unchecked") - public WfMgmtRunMsg build() { - try { - WfMgmtRunMsg record = new WfMgmtRunMsg(); - record.timestamp = - fieldSetFlags()[0] ? this.timestamp : (java.lang.Long) defaultValue(fields()[0]); - record.runId = - fieldSetFlags()[1] ? this.runId : (java.lang.String) defaultValue(fields()[1]); - record.state = - fieldSetFlags()[2] - ? this.state - : (org.icgc.argo.workflow_management.streams.schema.RunState) - defaultValue(fields()[2]); - record.workflowUrl = - fieldSetFlags()[3] ? this.workflowUrl : (java.lang.String) defaultValue(fields()[3]); - record.workflowType = - fieldSetFlags()[4] ? this.workflowType : (java.lang.String) defaultValue(fields()[4]); - record.workflowTypeVersion = - fieldSetFlags()[5] - ? this.workflowTypeVersion - : (java.lang.String) defaultValue(fields()[5]); - record.workflowParamsJsonStr = - fieldSetFlags()[6] - ? this.workflowParamsJsonStr - : (java.lang.String) defaultValue(fields()[6]); - if (workflowEngineParamsBuilder != null) { - try { - record.workflowEngineParams = this.workflowEngineParamsBuilder.build(); - } catch (org.apache.avro.AvroMissingFieldException e) { - e.addParentField(record.getSchema().getField("workflowEngineParams")); - throw e; - } - } else { - record.workflowEngineParams = - fieldSetFlags()[7] - ? this.workflowEngineParams - : (org.icgc.argo.workflow_management.streams.schema.EngineParams) - defaultValue(fields()[7]); - } - return record; - } catch (org.apache.avro.AvroMissingFieldException e) { - throw e; - } catch (java.lang.Exception e) { - throw new org.apache.avro.AvroRuntimeException(e); - } - } - } - - @SuppressWarnings("unchecked") - private static final org.apache.avro.io.DatumWriter WRITER$ = - (org.apache.avro.io.DatumWriter) MODEL$.createDatumWriter(SCHEMA$); - - @Override - public void writeExternal(java.io.ObjectOutput out) throws java.io.IOException { - WRITER$.write(this, SpecificData.getEncoder(out)); - } - - @SuppressWarnings("unchecked") - private static final org.apache.avro.io.DatumReader READER$ = - (org.apache.avro.io.DatumReader) MODEL$.createDatumReader(SCHEMA$); - - @Override - public void readExternal(java.io.ObjectInput in) throws java.io.IOException { - READER$.read(this, SpecificData.getDecoder(in)); - } - - @Override - protected boolean hasCustomCoders() { - return true; - } - - @Override - public void customEncode(org.apache.avro.io.Encoder out) throws java.io.IOException { - out.writeLong(this.timestamp); - - out.writeString(this.runId); - - out.writeEnum(this.state.ordinal()); - - if (this.workflowUrl == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.workflowUrl); - } - - if (this.workflowType == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.workflowType); - } - - if (this.workflowTypeVersion == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.workflowTypeVersion); - } - - if (this.workflowParamsJsonStr == null) { - out.writeIndex(0); - out.writeNull(); - } else { - out.writeIndex(1); - out.writeString(this.workflowParamsJsonStr); - } - - this.workflowEngineParams.customEncode(out); - } - - @Override - public void customDecode(org.apache.avro.io.ResolvingDecoder in) throws java.io.IOException { - org.apache.avro.Schema.Field[] fieldOrder = in.readFieldOrderIfDiff(); - if (fieldOrder == null) { - this.timestamp = in.readLong(); - - this.runId = in.readString(); - - this.state = - org.icgc.argo.workflow_management.streams.schema.RunState.values()[in.readEnum()]; - - if (in.readIndex() != 1) { - in.readNull(); - this.workflowUrl = null; - } else { - this.workflowUrl = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.workflowType = null; - } else { - this.workflowType = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.workflowTypeVersion = null; - } else { - this.workflowTypeVersion = in.readString(); - } - - if (in.readIndex() != 1) { - in.readNull(); - this.workflowParamsJsonStr = null; - } else { - this.workflowParamsJsonStr = in.readString(); - } - - if (this.workflowEngineParams == null) { - this.workflowEngineParams = - new org.icgc.argo.workflow_management.streams.schema.EngineParams(); - } - this.workflowEngineParams.customDecode(in); - - } else { - for (int i = 0; i < 8; i++) { - switch (fieldOrder[i].pos()) { - case 0: - this.timestamp = in.readLong(); - break; - - case 1: - this.runId = in.readString(); - break; - - case 2: - this.state = - org.icgc.argo.workflow_management.streams.schema.RunState.values()[in.readEnum()]; - break; - - case 3: - if (in.readIndex() != 1) { - in.readNull(); - this.workflowUrl = null; - } else { - this.workflowUrl = in.readString(); - } - break; - - case 4: - if (in.readIndex() != 1) { - in.readNull(); - this.workflowType = null; - } else { - this.workflowType = in.readString(); - } - break; - - case 5: - if (in.readIndex() != 1) { - in.readNull(); - this.workflowTypeVersion = null; - } else { - this.workflowTypeVersion = in.readString(); - } - break; - - case 6: - if (in.readIndex() != 1) { - in.readNull(); - this.workflowParamsJsonStr = null; - } else { - this.workflowParamsJsonStr = in.readString(); - } - break; - - case 7: - if (this.workflowEngineParams == null) { - this.workflowEngineParams = - new org.icgc.argo.workflow_management.streams.schema.EngineParams(); - } - this.workflowEngineParams.customDecode(in); - break; - - default: - throw new java.io.IOException("Corrupt ResolvingDecoder."); - } - } - } - } -} From dadf7ab20c89a4f999314b777908a61857abfe84 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 23 Aug 2021 17:09:40 -0400 Subject: [PATCH 06/26] update @nonnull handling --- .../util/VolumeMounts.java | 8 +++--- .../wes/properties/NextflowProperties.java | 3 +-- .../workflow_management/VolumeMountsTest.java | 25 +++++++++++++++---- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 26e2270..89e9bd4 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -31,10 +31,12 @@ public static Set extract( .flatMap( path -> Optional.ofNullable(k8sProperties.getVolMountMappings()) - .map(volMountsMapping -> extract(volMountsMapping, path)) - .orElse(Set.copyOf(k8sProperties.getVolMounts())).stream()) + .map(volMountsMapping -> extract(volMountsMapping, path).stream()) + .orElse(Stream.empty())) .collect(Collectors.toSet()); - return volMounts.isEmpty() ? k8sProperties.getVolMounts() : volMounts; + return volMounts.isEmpty() + ? Optional.ofNullable(k8sProperties.getVolMounts()).orElse(Set.of()) + : volMounts; } } diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index ef0ebda..cf0a363 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -20,7 +20,6 @@ import java.util.Set; import lombok.Data; -import lombok.NonNull; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; @@ -42,7 +41,7 @@ public static class K8sProperties { private String serviceAccount; private String namespace; private String runNamespace; - @NonNull private Set volMounts; + private Set volMounts; private Set volMountMappings; private String masterUrl; private boolean trustCertificate; diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index c44074c..ab21359 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -55,7 +55,8 @@ public void multiMatchTest() { @Test public void testExtractFromPropertiesAndEngineParams() { - val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); + val k8sProperties = new NextflowProperties.K8sProperties(); + k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); // include some inconsistent matching strings (with and without /) k8sProperties.setVolMountMappings( @@ -81,7 +82,7 @@ public void testExtractFromPropertiesAndEngineParams() { @Test public void testExtractFromPropertiesWhenMissingVolMountsMapping() { - val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); + val k8sProperties = new NextflowProperties.K8sProperties(); k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); val workflowEngineParams = @@ -97,9 +98,9 @@ public void testExtractFromPropertiesWhenMissingVolMountsMapping() { @Test public void testExtractFromPropertiesWhenMissingDirectories() { - val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); - k8sProperties.setVolMountMappings(Set.of("pv-claim:/some/dir,/some/dir/")); + val k8sProperties = new NextflowProperties.K8sProperties(); k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); + k8sProperties.setVolMountMappings(Set.of("pv-claim:/some/dir,/some/dir/")); val workflowEngineParams = WorkflowEngineParams.builder().build(); @@ -109,7 +110,7 @@ public void testExtractFromPropertiesWhenMissingDirectories() { @Test public void testExtractFromPropertiesDefaultVolMountsOnNoMatch() { - val k8sProperties = new NextflowProperties.K8sProperties(Set.of("pv-claim:/some/dir")); + val k8sProperties = new NextflowProperties.K8sProperties(); k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); val workflowEngineParams = @@ -122,4 +123,18 @@ public void testExtractFromPropertiesDefaultVolMountsOnNoMatch() { assertEquals( Set.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } + + @Test + public void testExtractFromPropertiesNullVolMountsDefault() { + val k8sProperties = new NextflowProperties.K8sProperties(); + + val workflowEngineParams = + WorkflowEngineParams.builder() + .launchDir("/some/dir/launch/dir") + .projectDir("/some/dir/project/dir") + .workDir("/some/dir/work/dir") + .build(); + + assertEquals(Set.of(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); + } } From 5e527668999a5e36f0bd90f37f70569657d74dcd Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 23 Aug 2021 17:11:42 -0400 Subject: [PATCH 07/26] test name --- .../org/icgc/argo/workflow_management/VolumeMountsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index ab21359..284cd67 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -125,7 +125,7 @@ public void testExtractFromPropertiesDefaultVolMountsOnNoMatch() { } @Test - public void testExtractFromPropertiesNullVolMountsDefault() { + public void testExtractFromPropertiesNullVolMountsAndNullMappingsDefault() { val k8sProperties = new NextflowProperties.K8sProperties(); val workflowEngineParams = From 9ed03af772726ebd82ec91cd7e1c343508ecee51 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Tue, 24 Aug 2021 10:31:23 -0400 Subject: [PATCH 08/26] nicer code --- .../util/VolumeMounts.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 89e9bd4..51fa0cd 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -7,7 +7,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.NonNull; -import lombok.val; import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; @@ -22,21 +21,18 @@ public static Set extract(@NonNull Set mappings, @NonNull String public static Set extract( NextflowProperties.K8sProperties k8sProperties, WorkflowEngineParams workflowEngineParams) { - val volMounts = - Stream.of( - workflowEngineParams.getLaunchDir(), - workflowEngineParams.getProjectDir(), - workflowEngineParams.getWorkDir()) - .filter(Objects::nonNull) - .flatMap( - path -> - Optional.ofNullable(k8sProperties.getVolMountMappings()) - .map(volMountsMapping -> extract(volMountsMapping, path).stream()) - .orElse(Stream.empty())) - .collect(Collectors.toSet()); - - return volMounts.isEmpty() - ? Optional.ofNullable(k8sProperties.getVolMounts()).orElse(Set.of()) - : volMounts; + return Optional.ofNullable(k8sProperties.getVolMountMappings()) + .map( + volMountsMapping -> + Stream.of( + workflowEngineParams.getLaunchDir(), + workflowEngineParams.getProjectDir(), + workflowEngineParams.getWorkDir()) + .filter(Objects::nonNull) + .flatMap(path -> extract(volMountsMapping, path).stream()) + .collect(Collectors.toSet())) + .flatMap(volMounts -> volMounts.isEmpty() ? Optional.empty() : Optional.of(volMounts)) + .or(() -> Optional.ofNullable(k8sProperties.getVolMounts())) + .orElse(Set.of()); } } From 7c590e2a9cfe65019df0646d066e3c9ed14309f5 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Tue, 24 Aug 2021 12:14:08 -0400 Subject: [PATCH 09/26] refactored to not need explicit mappings, rather just reads the existing --- .../util/VolumeMounts.java | 47 +++++--- .../wes/properties/NextflowProperties.java | 1 - src/main/resources/application.yml | 2 - .../workflow_management/VolumeMountsTest.java | 109 +++++++----------- 4 files changed, 70 insertions(+), 89 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 51fa0cd..9c4cbb9 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,38 +1,49 @@ package org.icgc.argo.workflow_management.util; +import lombok.NonNull; +import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; +import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; + import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.NonNull; -import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; -import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { - public static Set extract(@NonNull Set mappings, @NonNull String path) { - return mappings.stream() + public static Set extract(@NonNull Set volMounts, @NonNull String path) { + return extract(volMounts, Set.of(path)); + } + + public static Set extract(@NonNull Set volMounts, @NonNull Set paths) { + return volMounts.stream() .filter( - mapping -> Pattern.compile(Pattern.quote(mapping.split(",")[1])).matcher(path).find()) - .map(matchingMapping -> matchingMapping.split(",")[0]) + volMount -> + paths.stream() + .anyMatch( + path -> + // evaluate that the path in the engine param starts with volume mount + // path, remembering the format ... (volume-claim:volume-mount-path) + Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) + .matcher(path) + .find())) .collect(Collectors.toSet()); } public static Set extract( NextflowProperties.K8sProperties k8sProperties, WorkflowEngineParams workflowEngineParams) { - return Optional.ofNullable(k8sProperties.getVolMountMappings()) + return Optional.ofNullable(k8sProperties.getVolMounts()) .map( - volMountsMapping -> - Stream.of( - workflowEngineParams.getLaunchDir(), - workflowEngineParams.getProjectDir(), - workflowEngineParams.getWorkDir()) - .filter(Objects::nonNull) - .flatMap(path -> extract(volMountsMapping, path).stream()) - .collect(Collectors.toSet())) - .flatMap(volMounts -> volMounts.isEmpty() ? Optional.empty() : Optional.of(volMounts)) - .or(() -> Optional.ofNullable(k8sProperties.getVolMounts())) + volMounts -> + extract( + volMounts, + Stream.of( + workflowEngineParams.getLaunchDir(), + workflowEngineParams.getProjectDir(), + workflowEngineParams.getWorkDir()) + .filter(Objects::nonNull) + .collect(Collectors.toSet()))) .orElse(Set.of()); } } diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index cf0a363..5e26e67 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -42,7 +42,6 @@ public static class K8sProperties { private String namespace; private String runNamespace; private Set volMounts; - private Set volMountMappings; private String masterUrl; private boolean trustCertificate; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e457050..e10c223 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -6,8 +6,6 @@ nextflow: serviceAccount: "default" volMounts: - "pv-claim:/some/dir" - volMountMappings: - - "pv-claim:/some/dir,/some/dir" masterUrl: "localhost:8080" trustCertificate: false weblogUrl: "http://localhost" diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index 284cd67..0c7d50c 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -1,89 +1,91 @@ package org.icgc.argo.workflow_management; -import static org.junit.Assert.assertEquals; - -import java.util.Set; import lombok.val; import org.icgc.argo.workflow_management.util.VolumeMounts; import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; import org.junit.Test; +import java.util.Set; + +import static org.junit.Assert.assertEquals; + public class VolumeMountsTest { @Test public void defaultConfigTest() { - val mappings = Set.of("pv-claim:/some/dir,/some/dir/"); + val volMounts = Set.of("pv-claim:/some/dir"); assertEquals( - Set.of("pv-claim:/some/dir"), VolumeMounts.extract(mappings, "/some/dir/with/sub/dir")); + Set.of("pv-claim:/some/dir"), VolumeMounts.extract(volMounts, "/some/dir/with/sub/dir")); } @Test public void multiDirTest() { - // include some inconsistent matching strings (with and without /) - val mappings = + val volMounts = Set.of( - "pv-claim-one:/test-dir-1/,dir-one", - "pv-claim-two:/test-dir-2/,/dir-two", - "pv-claim-three:/test-dir-3/,dir-three/"); + "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3"); assertEquals( - Set.of("pv-claim-one:/test-dir-1/"), VolumeMounts.extract(mappings, "/dir-one/test/dir")); + Set.of("pv-claim-one:/test-dir-1"), VolumeMounts.extract(volMounts, "/test-dir-1/sub/dir")); assertEquals( - Set.of("pv-claim-two:/test-dir-2/"), VolumeMounts.extract(mappings, "/dir-two/test/dir")); + Set.of("pv-claim-two:/test-dir-2"), VolumeMounts.extract(volMounts, "/test-dir-2/sub/dir")); assertEquals( - Set.of("pv-claim-three:/test-dir-3/"), - VolumeMounts.extract(mappings, "/dir-three/test/dir")); + Set.of("pv-claim-three:/test-dir-3"), + VolumeMounts.extract(volMounts, "/test-dir-3/sub/dir")); } @Test - public void multiMatchTest() { - val mappings = - Set.of( - "pv-claim-one:/test-dir-1/,/vol-dir/", - "pv-claim-one:/test-dir-2/,/vol-dir/", - "pv-claim-one:/test-dir-3/,/vol-dir/"); + public void multiMatchAllTest() { + val volMounts = + Set.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); assertEquals( - Set.of( - "pv-claim-one:/test-dir-1/", "pv-claim-one:/test-dir-2/", "pv-claim-one:/test-dir-3/"), - VolumeMounts.extract(mappings, "/vol-dir/sub/dir")); + Set.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), + VolumeMounts.extract(volMounts, Set.of("/test-dir-1", "/test-dir-2", "/test-dir-3"))); + } + + @Test + public void multiMatchPartialTest() { + val volMounts = + Set.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); + + assertEquals( + Set.of("pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), + VolumeMounts.extract(volMounts, Set.of("/test-dir-2", "/test-dir-3"))); } @Test public void testExtractFromPropertiesAndEngineParams() { val k8sProperties = new NextflowProperties.K8sProperties(); - k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); - // include some inconsistent matching strings (with and without /) - k8sProperties.setVolMountMappings( + k8sProperties.setVolMounts( Set.of( - "pv-claim-one:/test-dir-1/,/dir-one/", - "pv-claim-two:/test-dir-2/,/dir-two", - "pv-claim-three:/test-dir-3/,dir-three/")); + "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3")); + // set a different volume for all three properties val workflowEngineParams = WorkflowEngineParams.builder() - .launchDir("/dir-one/test/dir") - .projectDir("/dir-two/test/dir") - .workDir("/dir-three/test/dir") + .launchDir("/test-dir-1/sub/dir") + .projectDir("/test-dir-2/sub/dir") + .workDir("/test-dir-3/sub/dir") .build(); assertEquals( Set.of( - "pv-claim-one:/test-dir-1/", - "pv-claim-two:/test-dir-2/", - "pv-claim-three:/test-dir-3/"), + "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @Test - public void testExtractFromPropertiesWhenMissingVolMountsMapping() { + public void testExtractFromPropertiesNoMatchVolMountsDefault() { val k8sProperties = new NextflowProperties.K8sProperties(); - k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); + + k8sProperties.setVolMounts( + Set.of( + "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3")); val workflowEngineParams = WorkflowEngineParams.builder() @@ -92,40 +94,11 @@ public void testExtractFromPropertiesWhenMissingVolMountsMapping() { .workDir("/some/dir/work/dir") .build(); - assertEquals( - Set.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); - } - - @Test - public void testExtractFromPropertiesWhenMissingDirectories() { - val k8sProperties = new NextflowProperties.K8sProperties(); - k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); - k8sProperties.setVolMountMappings(Set.of("pv-claim:/some/dir,/some/dir/")); - - val workflowEngineParams = WorkflowEngineParams.builder().build(); - - assertEquals( - Set.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); - } - - @Test - public void testExtractFromPropertiesDefaultVolMountsOnNoMatch() { - val k8sProperties = new NextflowProperties.K8sProperties(); - k8sProperties.setVolMounts(Set.of("pv-claim:/some/dir")); - - val workflowEngineParams = - WorkflowEngineParams.builder() - .launchDir("/different/dir/launch/dir") - .projectDir("/different/dir/project/dir") - .workDir("/different/dir/work/dir") - .build(); - - assertEquals( - Set.of("pv-claim:/some/dir"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); + assertEquals(Set.of(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @Test - public void testExtractFromPropertiesNullVolMountsAndNullMappingsDefault() { + public void testExtractFromPropertiesNullVolMountsDefault() { val k8sProperties = new NextflowProperties.K8sProperties(); val workflowEngineParams = From 05e44df713064736e81bfb983bbfdb4b742b809d Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Tue, 24 Aug 2021 12:37:50 -0400 Subject: [PATCH 10/26] add test for missing engine params --- .../util/VolumeMounts.java | 7 +++--- .../workflow_management/VolumeMountsTest.java | 23 +++++++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 9c4cbb9..533670e 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,15 +1,14 @@ package org.icgc.argo.workflow_management.util; -import lombok.NonNull; -import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; -import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; - import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.NonNull; +import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; +import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { public static Set extract(@NonNull Set volMounts, @NonNull String path) { diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index 0c7d50c..bdc2bb6 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -1,15 +1,14 @@ package org.icgc.argo.workflow_management; +import static org.junit.Assert.assertEquals; + +import java.util.Set; import lombok.val; import org.icgc.argo.workflow_management.util.VolumeMounts; import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; import org.junit.Test; -import java.util.Set; - -import static org.junit.Assert.assertEquals; - public class VolumeMountsTest { @Test @@ -79,6 +78,22 @@ public void testExtractFromPropertiesAndEngineParams() { VolumeMounts.extract(k8sProperties, workflowEngineParams)); } + @Test + public void testExtractFromPropertiesMissingEngineParams() { + val k8sProperties = new NextflowProperties.K8sProperties(); + + k8sProperties.setVolMounts( + Set.of( + "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3")); + + val workflowEngineParams = + WorkflowEngineParams.builder().workDir("/test-dir-3/sub/dir").build(); + + assertEquals( + Set.of("pv-claim-three:/test-dir-3"), + VolumeMounts.extract(k8sProperties, workflowEngineParams)); + } + @Test public void testExtractFromPropertiesNoMatchVolMountsDefault() { val k8sProperties = new NextflowProperties.K8sProperties(); From b251e463139410555b90d9bd0ead9d8c45e88c4c Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Tue, 24 Aug 2021 12:58:19 -0400 Subject: [PATCH 11/26] edge case for no engine params set + comments --- .../util/VolumeMounts.java | 36 +++++++++++-------- .../wes/properties/NextflowProperties.java | 4 +-- .../workflow_management/VolumeMountsTest.java | 34 +++++++++++++----- 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 533670e..5aaece7 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,5 +1,6 @@ package org.icgc.argo.workflow_management.util; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -11,23 +12,30 @@ import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { - public static Set extract(@NonNull Set volMounts, @NonNull String path) { + public static Set extract(@NonNull List volMounts, @NonNull String path) { return extract(volMounts, Set.of(path)); } - public static Set extract(@NonNull Set volMounts, @NonNull Set paths) { - return volMounts.stream() - .filter( - volMount -> - paths.stream() - .anyMatch( - path -> - // evaluate that the path in the engine param starts with volume mount - // path, remembering the format ... (volume-claim:volume-mount-path) - Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) - .matcher(path) - .find())) - .collect(Collectors.toSet()); + public static Set extract(@NonNull List volMounts, @NonNull Set paths) { + // when no engine params are provided, Nextflow will by default use the first volume claim in + // the list provided, we need to ensure at least one claim is sent even when not explicitly + // mentioned, meaning that if we are matching against and empty set (no engine params in the + // request) we need to provide at least one volume mount for Nextflow to use as the default + return paths.isEmpty() + ? Set.of(volMounts.get(0)) + : volMounts.stream() + .filter( + volMount -> + paths.stream() + .anyMatch( + path -> + // evaluate that the path in the engine param starts with volume + // mount path, remembering the format ... + // (volume-claim:volume-mount-path) + Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) + .matcher(path) + .find())) + .collect(Collectors.toSet()); } public static Set extract( diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index 5e26e67..a17093f 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -18,7 +18,7 @@ package org.icgc.argo.workflow_management.wes.properties; -import java.util.Set; +import java.util.List; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -41,7 +41,7 @@ public static class K8sProperties { private String serviceAccount; private String namespace; private String runNamespace; - private Set volMounts; + private List volMounts; private String masterUrl; private boolean trustCertificate; } diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index bdc2bb6..d7512f3 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -2,6 +2,7 @@ import static org.junit.Assert.assertEquals; +import java.util.List; import java.util.Set; import lombok.val; import org.icgc.argo.workflow_management.util.VolumeMounts; @@ -13,7 +14,7 @@ public class VolumeMountsTest { @Test public void defaultConfigTest() { - val volMounts = Set.of("pv-claim:/some/dir"); + val volMounts = List.of("pv-claim:/some/dir"); assertEquals( Set.of("pv-claim:/some/dir"), VolumeMounts.extract(volMounts, "/some/dir/with/sub/dir")); @@ -22,7 +23,7 @@ public void defaultConfigTest() { @Test public void multiDirTest() { val volMounts = - Set.of( + List.of( "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3"); assertEquals( @@ -39,7 +40,7 @@ public void multiDirTest() { @Test public void multiMatchAllTest() { val volMounts = - Set.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); + List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); assertEquals( Set.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), @@ -49,7 +50,7 @@ public void multiMatchAllTest() { @Test public void multiMatchPartialTest() { val volMounts = - Set.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); + List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); assertEquals( Set.of("pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), @@ -61,7 +62,7 @@ public void testExtractFromPropertiesAndEngineParams() { val k8sProperties = new NextflowProperties.K8sProperties(); k8sProperties.setVolMounts( - Set.of( + List.of( "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3")); // set a different volume for all three properties @@ -79,11 +80,11 @@ public void testExtractFromPropertiesAndEngineParams() { } @Test - public void testExtractFromPropertiesMissingEngineParams() { + public void testExtractFromPropertiesMissingSomeEngineParams() { val k8sProperties = new NextflowProperties.K8sProperties(); k8sProperties.setVolMounts( - Set.of( + List.of( "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3")); val workflowEngineParams = @@ -94,12 +95,29 @@ public void testExtractFromPropertiesMissingEngineParams() { VolumeMounts.extract(k8sProperties, workflowEngineParams)); } + @Test + public void testExtractFromPropertiesMissingAllEngineParams() { + val k8sProperties = new NextflowProperties.K8sProperties(); + + k8sProperties.setVolMounts( + List.of( + "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3")); + + val workflowEngineParams = WorkflowEngineParams.builder().build(); + + // when no engine params are provided, Nextflow will be default use the first volume claim in + // the list provided, we need to ensure at least one claim is sent even when not explicitly set + assertEquals( + Set.of("pv-claim-one:/test-dir-1"), + VolumeMounts.extract(k8sProperties, workflowEngineParams)); + } + @Test public void testExtractFromPropertiesNoMatchVolMountsDefault() { val k8sProperties = new NextflowProperties.K8sProperties(); k8sProperties.setVolMounts( - Set.of( + List.of( "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3")); val workflowEngineParams = From 986959779b1d867256e9fd2e88577a180b3facc5 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Tue, 24 Aug 2021 13:46:29 -0400 Subject: [PATCH 12/26] volMounts should be List --- .../util/VolumeMounts.java | 18 ++++---- .../workflow_management/VolumeMountsTest.java | 42 ++++++++++++------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 5aaece7..42fa5f0 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,9 +1,9 @@ package org.icgc.argo.workflow_management.util; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -12,17 +12,17 @@ import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { - public static Set extract(@NonNull List volMounts, @NonNull String path) { - return extract(volMounts, Set.of(path)); + public static List extract(@NonNull List volMounts, @NonNull String path) { + return extract(volMounts, List.of(path)); } - public static Set extract(@NonNull List volMounts, @NonNull Set paths) { + public static List extract(@NonNull List volMounts, @NonNull List paths) { // when no engine params are provided, Nextflow will by default use the first volume claim in // the list provided, we need to ensure at least one claim is sent even when not explicitly // mentioned, meaning that if we are matching against and empty set (no engine params in the // request) we need to provide at least one volume mount for Nextflow to use as the default return paths.isEmpty() - ? Set.of(volMounts.get(0)) + ? List.of(volMounts.get(0)) : volMounts.stream() .filter( volMount -> @@ -35,10 +35,10 @@ public static Set extract(@NonNull List volMounts, @NonNull Set< Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) .matcher(path) .find())) - .collect(Collectors.toSet()); + .collect(Collectors.toList()); } - public static Set extract( + public static List extract( NextflowProperties.K8sProperties k8sProperties, WorkflowEngineParams workflowEngineParams) { return Optional.ofNullable(k8sProperties.getVolMounts()) .map( @@ -50,7 +50,7 @@ public static Set extract( workflowEngineParams.getProjectDir(), workflowEngineParams.getWorkDir()) .filter(Objects::nonNull) - .collect(Collectors.toSet()))) - .orElse(Set.of()); + .collect(Collectors.toList()))) + .orElse(Collections.emptyList()); } } diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index d7512f3..43e39c1 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -2,8 +2,8 @@ import static org.junit.Assert.assertEquals; +import java.util.Collections; import java.util.List; -import java.util.Set; import lombok.val; import org.icgc.argo.workflow_management.util.VolumeMounts; import org.icgc.argo.workflow_management.wes.model.WorkflowEngineParams; @@ -17,7 +17,7 @@ public void defaultConfigTest() { val volMounts = List.of("pv-claim:/some/dir"); assertEquals( - Set.of("pv-claim:/some/dir"), VolumeMounts.extract(volMounts, "/some/dir/with/sub/dir")); + List.of("pv-claim:/some/dir"), VolumeMounts.extract(volMounts, "/some/dir/with/sub/dir")); } @Test @@ -27,13 +27,15 @@ public void multiDirTest() { "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3"); assertEquals( - Set.of("pv-claim-one:/test-dir-1"), VolumeMounts.extract(volMounts, "/test-dir-1/sub/dir")); + List.of("pv-claim-one:/test-dir-1"), + VolumeMounts.extract(volMounts, "/test-dir-1/sub/dir")); assertEquals( - Set.of("pv-claim-two:/test-dir-2"), VolumeMounts.extract(volMounts, "/test-dir-2/sub/dir")); + List.of("pv-claim-two:/test-dir-2"), + VolumeMounts.extract(volMounts, "/test-dir-2/sub/dir")); assertEquals( - Set.of("pv-claim-three:/test-dir-3"), + List.of("pv-claim-three:/test-dir-3"), VolumeMounts.extract(volMounts, "/test-dir-3/sub/dir")); } @@ -43,8 +45,8 @@ public void multiMatchAllTest() { List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); assertEquals( - Set.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), - VolumeMounts.extract(volMounts, Set.of("/test-dir-1", "/test-dir-2", "/test-dir-3"))); + List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), + VolumeMounts.extract(volMounts, List.of("/test-dir-1", "/test-dir-2", "/test-dir-3"))); } @Test @@ -53,8 +55,18 @@ public void multiMatchPartialTest() { List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); assertEquals( - Set.of("pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), - VolumeMounts.extract(volMounts, Set.of("/test-dir-2", "/test-dir-3"))); + List.of("pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"), + VolumeMounts.extract(volMounts, List.of("/test-dir-2", "/test-dir-3"))); + } + + @Test + public void multiMatchNoDuplicates() { + val volMounts = + List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); + + assertEquals( + List.of("pv-claim-one:/test-dir-1"), + VolumeMounts.extract(volMounts, List.of("/test-dir-1", "/test-dir-1", "/test-dir-1"))); } @Test @@ -74,7 +86,7 @@ public void testExtractFromPropertiesAndEngineParams() { .build(); assertEquals( - Set.of( + List.of( "pv-claim-one:/test-dir-1", "pv-claim-two:/test-dir-2", "pv-claim-three:/test-dir-3"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @@ -91,7 +103,7 @@ public void testExtractFromPropertiesMissingSomeEngineParams() { WorkflowEngineParams.builder().workDir("/test-dir-3/sub/dir").build(); assertEquals( - Set.of("pv-claim-three:/test-dir-3"), + List.of("pv-claim-three:/test-dir-3"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @@ -108,7 +120,7 @@ public void testExtractFromPropertiesMissingAllEngineParams() { // when no engine params are provided, Nextflow will be default use the first volume claim in // the list provided, we need to ensure at least one claim is sent even when not explicitly set assertEquals( - Set.of("pv-claim-one:/test-dir-1"), + List.of("pv-claim-one:/test-dir-1"), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @@ -127,7 +139,8 @@ public void testExtractFromPropertiesNoMatchVolMountsDefault() { .workDir("/some/dir/work/dir") .build(); - assertEquals(Set.of(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); + assertEquals( + Collections.emptyList(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @Test @@ -141,6 +154,7 @@ public void testExtractFromPropertiesNullVolMountsDefault() { .workDir("/some/dir/work/dir") .build(); - assertEquals(Set.of(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); + assertEquals( + Collections.emptyList(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); } } From a38da94e51234c1575a0402405bc9b6a8cfa4195 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Tue, 24 Aug 2021 13:51:15 -0400 Subject: [PATCH 13/26] distinct list handling/test --- .../util/VolumeMounts.java | 35 ++++++++++--------- .../workflow_management/VolumeMountsTest.java | 2 +- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 42fa5f0..a532f47 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -12,8 +12,20 @@ import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { - public static List extract(@NonNull List volMounts, @NonNull String path) { - return extract(volMounts, List.of(path)); + public static List extract( + NextflowProperties.K8sProperties k8sProperties, WorkflowEngineParams workflowEngineParams) { + return Optional.ofNullable(k8sProperties.getVolMounts()) + .map( + volMounts -> + extract( + volMounts, + Stream.of( + workflowEngineParams.getLaunchDir(), + workflowEngineParams.getProjectDir(), + workflowEngineParams.getWorkDir()) + .filter(Objects::nonNull) + .collect(Collectors.toList()))) + .orElse(Collections.emptyList()); } public static List extract(@NonNull List volMounts, @NonNull List paths) { @@ -35,22 +47,13 @@ public static List extract(@NonNull List volMounts, @NonNull Lis Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) .matcher(path) .find())) + // volMounts need to be provided as a list, + // however we only care about distinct values + .distinct() .collect(Collectors.toList()); } - public static List extract( - NextflowProperties.K8sProperties k8sProperties, WorkflowEngineParams workflowEngineParams) { - return Optional.ofNullable(k8sProperties.getVolMounts()) - .map( - volMounts -> - extract( - volMounts, - Stream.of( - workflowEngineParams.getLaunchDir(), - workflowEngineParams.getProjectDir(), - workflowEngineParams.getWorkDir()) - .filter(Objects::nonNull) - .collect(Collectors.toList()))) - .orElse(Collections.emptyList()); + public static List extract(@NonNull List volMounts, @NonNull String path) { + return extract(volMounts, List.of(path)); } } diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index 43e39c1..e2167ec 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -62,7 +62,7 @@ public void multiMatchPartialTest() { @Test public void multiMatchNoDuplicates() { val volMounts = - List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-2", "pv-claim-one:/test-dir-3"); + List.of("pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-1", "pv-claim-one:/test-dir-1"); assertEquals( List.of("pv-claim-one:/test-dir-1"), From bce105a4ed6c974fca85faeb47de3b2ee7eaf429 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Tue, 24 Aug 2021 16:29:33 -0400 Subject: [PATCH 14/26] if not match we still want a default volMount if possible --- .../util/VolumeMounts.java | 50 +++++++++++-------- .../workflow_management/VolumeMountsTest.java | 3 +- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index a532f47..db0f05d 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,31 +30,36 @@ public static List extract( } public static List extract(@NonNull List volMounts, @NonNull List paths) { - // when no engine params are provided, Nextflow will by default use the first volume claim in - // the list provided, we need to ensure at least one claim is sent even when not explicitly - // mentioned, meaning that if we are matching against and empty set (no engine params in the - // request) we need to provide at least one volume mount for Nextflow to use as the default - return paths.isEmpty() - ? List.of(volMounts.get(0)) - : volMounts.stream() - .filter( - volMount -> - paths.stream() - .anyMatch( - path -> - // evaluate that the path in the engine param starts with volume - // mount path, remembering the format ... - // (volume-claim:volume-mount-path) - Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) - .matcher(path) - .find())) - // volMounts need to be provided as a list, - // however we only care about distinct values - .distinct() - .collect(Collectors.toList()); + return volMounts.stream() + .filter( + volMount -> + paths.stream() + .anyMatch( + path -> + // evaluate that the path in the engine param starts with volume + // mount path, remembering the format ... + // (volume-claim:volume-mount-path) + Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) + .matcher(path) + .find())) + // volMounts need to be provided as a list, + // however we only care about distinct values + .distinct() + // when no engine params are provided, Nextflow will by default use the first volume claim + // in the list provided, we need to ensure at least one claim is sent even when not + // explicitly mentioned, meaning that if we are matching against and empty set (no engine + // params in the request) we need to provide at least one volume mount for Nextflow to use + // as the default + .collect( + Collectors.collectingAndThen( + Collectors.toList(), + matchedVolMounts -> + matchedVolMounts.isEmpty() ? List.of(volMounts.get(0)) : matchedVolMounts)); } public static List extract(@NonNull List volMounts, @NonNull String path) { return extract(volMounts, List.of(path)); } + + private static final Predicate> isPathsEmpty = List::isEmpty; } diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index e2167ec..75242f3 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -140,7 +140,8 @@ public void testExtractFromPropertiesNoMatchVolMountsDefault() { .build(); assertEquals( - Collections.emptyList(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); + List.of("pv-claim-one:/test-dir-1"), + VolumeMounts.extract(k8sProperties, workflowEngineParams)); } @Test From 2ea2e419a713e1d7af0c91acba4005a7c3231928 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Wed, 25 Aug 2021 11:46:04 -0400 Subject: [PATCH 15/26] refactored, commented, and now throwing exception for no volMounts --- .../util/VolumeMounts.java | 100 ++++++++++++++---- .../workflow_management/VolumeMountsTest.java | 15 ++- 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index db0f05d..1710042 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,10 +1,11 @@ package org.icgc.argo.workflow_management.util; -import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; +import java.util.function.UnaryOperator; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -13,6 +14,16 @@ import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { + private static final HashMap volMountPatternCache = new HashMap<>(); + + /** + * Given the k8sProperties configured in application properties, and the workflow engine + * parameters for a run, extract the required volume mounts + * + * @param k8sProperties - properties containing the configured volume mounts + * @param workflowEngineParams - the workflow engine parameters for the run + * @return a list of volume mounts that are required for the engine parameters for the run + */ public static List extract( NextflowProperties.K8sProperties k8sProperties, WorkflowEngineParams workflowEngineParams) { return Optional.ofNullable(k8sProperties.getVolMounts()) @@ -26,40 +37,85 @@ public static List extract( workflowEngineParams.getWorkDir()) .filter(Objects::nonNull) .collect(Collectors.toList()))) - .orElse(Collections.emptyList()); + .orElseThrow( + () -> + new IllegalStateException( + "At lease on volMount must be configured in order for Nextflow to run in Kubernetes")); } + /** + * Given a list of volMounts and paths, returns the volMounts required for the paths provided + * where a volMount (with the format claim-name:mount-path) is considered required if the path + * starts with the volume mount path. If no matches are found and there is at least one volMount, + * the first volMount will be returned as the single item in a list as at least one volMount is + * required + * + * @param volMounts - list of volMounts to search through + * @param paths - list of paths to use as the search subject + * @return the matching volMounts given the search + */ public static List extract(@NonNull List volMounts, @NonNull List paths) { return volMounts.stream() - .filter( - volMount -> - paths.stream() - .anyMatch( - path -> - // evaluate that the path in the engine param starts with volume - // mount path, remembering the format ... - // (volume-claim:volume-mount-path) - Pattern.compile("^" + Pattern.quote(volMount.split(":")[1])) - .matcher(path) - .find())) + .filter(volMount -> paths.stream().anyMatch(getPathMatchFunctionForVolMount(volMount))) // volMounts need to be provided as a list, // however we only care about distinct values .distinct() - // when no engine params are provided, Nextflow will by default use the first volume claim - // in the list provided, we need to ensure at least one claim is sent even when not - // explicitly mentioned, meaning that if we are matching against and empty set (no engine - // params in the request) we need to provide at least one volume mount for Nextflow to use - // as the default .collect( Collectors.collectingAndThen( - Collectors.toList(), - matchedVolMounts -> - matchedVolMounts.isEmpty() ? List.of(volMounts.get(0)) : matchedVolMounts)); + Collectors.toList(), getDefaultVolMountIfEmptyFunctionForVolMount(volMounts))); } + /** + * Utility overload method that calls extract(@NonNull List volMounts, @NonNull + * List paths) + * + * @param volMounts - list of volMounts to search through + * @param path - the path to use as the search subject + * @return the matching volMounts given the search + */ public static List extract(@NonNull List volMounts, @NonNull String path) { return extract(volMounts, List.of(path)); } - private static final Predicate> isPathsEmpty = List::isEmpty; + /** + * Generates a function for matching path to volume mount + * + * @param volMount a single volMount entry from the list in application properties + * @return a function which evaluates that the path provided matches the pattern for the volMount + * used to generate this function + */ + private static Predicate getPathMatchFunctionForVolMount(String volMount) { + return path -> getPatternForVolMount(volMount).matcher(path).find(); + } + + /** + * Lazily computes a pattern for each volume mount (given the format + * volume-claim:volume-mount-path) in the application properties, backed by the static + * patternCache HashMap + * + * @param volMount - the volume mount to extract the pattern for + * @return the Pattern for the provided volMount string + */ + private static Pattern getPatternForVolMount(String volMount) { + return volMountPatternCache.computeIfAbsent( + volMount, key -> Pattern.compile("^" + Pattern.quote(volMount.split(":")[1]))); + } + + /** + * When no engine params are provided, Nextflow will by default use the first volume claim in the + * list provided, we need to ensure at least one such volume mount claim is sent even when not + * explicitly mentioned, meaning that if we are matching against and empty set (no engine params + * in the request) we need to provide at least one volume mount for Nextflow to use as the + * default. This method returns a function that checks if the matched volume mounts is empty and + * if so returns a list containing the first volMount that is configured for the application + * + * @param volMounts - the configured volume mounts in the application properties + * @return a function that checks the matched volume list for emptiness and returns a list + * containing the first volMount from the volMounts parameter + */ + private static UnaryOperator> getDefaultVolMountIfEmptyFunctionForVolMount( + List volMounts) { + return matchedVolMounts -> + matchedVolMounts.isEmpty() ? List.of(volMounts.get(0)) : matchedVolMounts; + } } diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index 75242f3..42ac09b 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -1,8 +1,9 @@ package org.icgc.argo.workflow_management; +import static graphql.Assert.assertTrue; import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; -import java.util.Collections; import java.util.List; import lombok.val; import org.icgc.argo.workflow_management.util.VolumeMounts; @@ -155,7 +156,15 @@ public void testExtractFromPropertiesNullVolMountsDefault() { .workDir("/some/dir/work/dir") .build(); - assertEquals( - Collections.emptyList(), VolumeMounts.extract(k8sProperties, workflowEngineParams)); + val exception = + assertThrows( + IllegalStateException.class, + () -> VolumeMounts.extract(k8sProperties, workflowEngineParams)); + + assertTrue( + exception + .getMessage() + .contains( + "At lease on volMount must be configured in order for Nextflow to run in Kubernetes")); } } From ed3c4042a98d2f824ba45bf346e7b9305ffc2add Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Wed, 25 Aug 2021 11:50:18 -0400 Subject: [PATCH 16/26] msg typo --- .../org/icgc/argo/workflow_management/util/VolumeMounts.java | 2 +- .../org/icgc/argo/workflow_management/VolumeMountsTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index 1710042..a660494 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -40,7 +40,7 @@ public static List extract( .orElseThrow( () -> new IllegalStateException( - "At lease on volMount must be configured in order for Nextflow to run in Kubernetes")); + "At least one volMount must be configured in order for Nextflow to run in Kubernetes")); } /** diff --git a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java index 42ac09b..f9ee3a8 100644 --- a/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java +++ b/src/test/java/org/icgc/argo/workflow_management/VolumeMountsTest.java @@ -165,6 +165,6 @@ public void testExtractFromPropertiesNullVolMountsDefault() { exception .getMessage() .contains( - "At lease on volMount must be configured in order for Nextflow to run in Kubernetes")); + "At least one volMount must be configured in order for Nextflow to run in Kubernetes")); } } From 5adefff56e78fc2f645ce3ab9e66180c83671ab4 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Wed, 25 Aug 2021 12:05:32 -0400 Subject: [PATCH 17/26] simplified --- .../util/VolumeMounts.java | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java index a660494..adea67f 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java +++ b/src/main/java/org/icgc/argo/workflow_management/util/VolumeMounts.java @@ -1,12 +1,10 @@ package org.icgc.argo.workflow_management.util; -import java.util.HashMap; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.function.Predicate; import java.util.function.UnaryOperator; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.NonNull; @@ -14,8 +12,6 @@ import org.icgc.argo.workflow_management.wes.properties.NextflowProperties; public class VolumeMounts { - private static final HashMap volMountPatternCache = new HashMap<>(); - /** * Given the k8sProperties configured in application properties, and the workflow engine * parameters for a run, extract the required volume mounts @@ -85,20 +81,7 @@ public static List extract(@NonNull List volMounts, @NonNull Str * used to generate this function */ private static Predicate getPathMatchFunctionForVolMount(String volMount) { - return path -> getPatternForVolMount(volMount).matcher(path).find(); - } - - /** - * Lazily computes a pattern for each volume mount (given the format - * volume-claim:volume-mount-path) in the application properties, backed by the static - * patternCache HashMap - * - * @param volMount - the volume mount to extract the pattern for - * @return the Pattern for the provided volMount string - */ - private static Pattern getPatternForVolMount(String volMount) { - return volMountPatternCache.computeIfAbsent( - volMount, key -> Pattern.compile("^" + Pattern.quote(volMount.split(":")[1]))); + return path -> path.startsWith(volMount.split(":")[1]); } /** From a1e9f2092575fb095ac089f2d39eaa0f69f42fd7 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Wed, 25 Aug 2021 15:25:38 -0400 Subject: [PATCH 18/26] Nextflow version: 21.04.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8b8251c..fc86580 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ io.nextflow nextflow - 20.07.1 + 21.04.3 From 95fc7ccf11dcae862806fccad453e7162e125530 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Wed, 25 Aug 2021 15:25:38 -0400 Subject: [PATCH 19/26] Revert "Nextflow version: 21.04.3" This reverts commit a1e9f2092575fb095ac089f2d39eaa0f69f42fd7. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fc86580..8b8251c 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,7 @@ io.nextflow nextflow - 21.04.3 + 20.07.1 From e46c3ac0ed290a59cb179eae93c6bb1dfe075812 Mon Sep 17 00:00:00 2001 From: jaserud Date: Wed, 25 Aug 2021 18:53:19 -0400 Subject: [PATCH 20/26] updated nextflow, rabbitmq, kafka, and springboot versions (#182) * updated nextflow, rabbitmq, kafka, and springboot versions * update nextflow version for real --- pom.xml | 33 +--- .../config/security/AuthDisabledConfig.java | 4 - .../config/security/AuthEnabledConfig.java | 178 ------------------ .../config/security/AuthProperties.java | 51 ----- .../graphql/model/GqlSearchQueryArgs.java | 2 +- .../wes/model/WorkflowEngineParams.java | 2 +- src/main/resources/application.yml | 11 -- 7 files changed, 7 insertions(+), 274 deletions(-) delete mode 100644 src/main/java/org/icgc/argo/workflow_management/config/security/AuthEnabledConfig.java delete mode 100644 src/main/java/org/icgc/argo/workflow_management/config/security/AuthProperties.java diff --git a/pom.xml b/pom.xml index 8b8251c..5975f23 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.2.2.RELEASE + 2.5.4 org.icgc.argo @@ -25,11 +25,6 @@ false - - springfox - springfox - http://oss.jfrog.org/artifactory/oss-snapshot-local/ - jcenter https://jcenter.bintray.com/ @@ -38,8 +33,7 @@ 11 - 3.0.0-SNAPSHOT - 0.0.7 + 0.0.9 @@ -68,7 +62,7 @@ io.nextflow nextflow - 20.07.1 + 21.04.3 @@ -84,27 +78,10 @@ - - org.springframework.boot - spring-boot-starter-security - - - org.springframework.boot - spring-boot-starter-oauth2-resource-server - - - org.springframework.security - spring-security-oauth2-jose - 5.3.3.RELEASE - - - org.springframework.security - spring-security-oauth2-client - org.springframework.security.oauth.boot spring-security-oauth2-autoconfigure - 2.0.2.RELEASE + 2.5.2 @@ -155,7 +132,7 @@ org.springframework.cloud spring-cloud-starter-stream-kafka - 3.0.1.RELEASE + 3.1.3 diff --git a/src/main/java/org/icgc/argo/workflow_management/config/security/AuthDisabledConfig.java b/src/main/java/org/icgc/argo/workflow_management/config/security/AuthDisabledConfig.java index 9cf478d..311d301 100644 --- a/src/main/java/org/icgc/argo/workflow_management/config/security/AuthDisabledConfig.java +++ b/src/main/java/org/icgc/argo/workflow_management/config/security/AuthDisabledConfig.java @@ -18,16 +18,12 @@ package org.icgc.argo.workflow_management.config.security; -import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Profile; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; import org.springframework.security.web.server.SecurityWebFilterChain; @EnableWebFluxSecurity -@Slf4j -@Profile("!secure") public class AuthDisabledConfig { @Bean public SecurityWebFilterChain securityFilterChain(ServerHttpSecurity http) { diff --git a/src/main/java/org/icgc/argo/workflow_management/config/security/AuthEnabledConfig.java b/src/main/java/org/icgc/argo/workflow_management/config/security/AuthEnabledConfig.java deleted file mode 100644 index 66326a4..0000000 --- a/src/main/java/org/icgc/argo/workflow_management/config/security/AuthEnabledConfig.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2021 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of the GNU Affero General Public License v3.0. - * You should have received a copy of the GNU Affero General Public License along with - * this program. If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.icgc.argo.workflow_management.config.security; - -import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; - -import com.google.common.collect.ImmutableList; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.security.KeyFactory; -import java.security.interfaces.RSAPublicKey; -import java.security.spec.X509EncodedKeySpec; -import java.util.Base64; -import java.util.Collection; -import java.util.Map; -import java.util.Objects; -import java.util.function.Function; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Profile; -import org.springframework.core.convert.converter.Converter; -import org.springframework.core.io.ResourceLoader; -import org.springframework.security.authentication.AbstractAuthenticationToken; -import org.springframework.security.config.annotation.method.configuration.EnableReactiveMethodSecurity; -import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; -import org.springframework.security.config.web.server.ServerHttpSecurity; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.GrantedAuthority; -import org.springframework.security.core.authority.SimpleGrantedAuthority; -import org.springframework.security.oauth2.jwt.Jwt; -import org.springframework.security.oauth2.jwt.NimbusReactiveJwtDecoder; -import org.springframework.security.oauth2.jwt.ReactiveJwtDecoder; -import org.springframework.security.oauth2.server.resource.authentication.JwtAuthenticationConverter; -import org.springframework.security.oauth2.server.resource.authentication.ReactiveJwtAuthenticationConverterAdapter; -import org.springframework.security.web.server.SecurityWebFilterChain; -import reactor.core.publisher.Mono; - -@EnableWebFluxSecurity -@Slf4j -@Profile("secure") -@EnableReactiveMethodSecurity -public class AuthEnabledConfig { - - private final AuthProperties authProperties; - - private final ResourceLoader resourceLoader; - - @Autowired - public AuthEnabledConfig(AuthProperties authProperties, ResourceLoader resourceLoader) { - this.authProperties = authProperties; - this.resourceLoader = resourceLoader; - } - - @Bean - public SecurityWebFilterChain securityFilterChain(ServerHttpSecurity http) { - http.csrf() - .disable() - .authorizeExchange() - .pathMatchers("/graphql/**") - .permitAll() - .pathMatchers("/actuator/**") - .permitAll() - .pathMatchers("/runs/**") - .permitAll() - .pathMatchers( - "/v2/api-docs", - "/configuration/ui", - "/swagger-resources/**", - "/configuration/security", - "/swagger-ui.html", - "/webjars/**") - .permitAll() - .and() - .authorizeExchange() - .anyExchange() - .authenticated() - .and() - .oauth2ResourceServer() - .jwt() - .jwtDecoder(jwtDecoder()) - .jwtAuthenticationConverter(grantedAuthoritiesExtractor()); - return http.build(); - } - - private Converter> grantedAuthoritiesExtractor() { - JwtAuthenticationConverter jwtAuthenticationConverter = new JwtAuthenticationConverter(); - jwtAuthenticationConverter.setJwtGrantedAuthoritiesConverter( - this.jwtToGrantedAuthoritiesConverter); - return new ReactiveJwtAuthenticationConverterAdapter(jwtAuthenticationConverter); - } - - private final Converter> jwtToGrantedAuthoritiesConverter = - (jwt) -> { - val scopesBuilder = ImmutableList.builder(); - - try { - val context = (Map) jwt.getClaims().get("context"); - scopesBuilder.addAll((Collection) context.get("scope")); - } catch (Exception e) { - log.error("Unable to extract scopes from JWT"); - } - - return scopesBuilder.build().stream().map(SimpleGrantedAuthority::new).collect(toList()); - }; - - @SneakyThrows - private ReactiveJwtDecoder jwtDecoder() { - String publicKeyStr; - - val publicKeyUrl = authProperties.getJwtPublicKeyUrl(); - if (publicKeyUrl != null && !publicKeyUrl.isEmpty()) { - publicKeyStr = fetchJWTPublicKey(publicKeyUrl); - } else { - publicKeyStr = authProperties.getJwtPublicKeyStr(); - } - - val publicKeyContent = - publicKeyStr - .replaceAll("\\n", "") - .replace("-----BEGIN PUBLIC KEY-----", "") - .replace("-----END PUBLIC KEY-----", ""); - - KeyFactory kf = KeyFactory.getInstance("RSA"); - X509EncodedKeySpec keySpecX509 = - new X509EncodedKeySpec(Base64.getDecoder().decode(publicKeyContent)); - RSAPublicKey publicKey = (RSAPublicKey) kf.generatePublic(keySpecX509); - - return NimbusReactiveJwtDecoder.withPublicKey(publicKey).build(); - } - - @SneakyThrows - private String fetchJWTPublicKey(String publicKeyUrl) { - log.info("Fetching EGO public key"); - val publicKeyResource = resourceLoader.getResource(publicKeyUrl); - - val stringBuilder = new StringBuilder(); - val reader = new BufferedReader(new InputStreamReader(publicKeyResource.getInputStream())); - - reader.lines().forEach(stringBuilder::append); - return stringBuilder.toString(); - } - - @Bean - public Function queryAndMutationScopeChecker() { - val expectedScopes = authProperties.getGraphqlScopes().getQueryAndMutation(); - return authentication -> { - val scopes = - authentication.getAuthorities().stream() - .map(Objects::toString) - .collect(toUnmodifiableList()); - - val foundScopes = - scopes.stream().filter(expectedScopes::contains).collect(toUnmodifiableList()); - - return foundScopes.size() > 0; - }; - } -} diff --git a/src/main/java/org/icgc/argo/workflow_management/config/security/AuthProperties.java b/src/main/java/org/icgc/argo/workflow_management/config/security/AuthProperties.java deleted file mode 100644 index d848472..0000000 --- a/src/main/java/org/icgc/argo/workflow_management/config/security/AuthProperties.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2021 The Ontario Institute for Cancer Research. All rights reserved - * - * This program and the accompanying materials are made available under the terms of the GNU Affero General Public License v3.0. - * You should have received a copy of the GNU Affero General Public License along with - * this program. If not, see . - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT - * SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; - * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER - * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -package org.icgc.argo.workflow_management.config.security; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import lombok.Data; -import lombok.Value; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.ConstructorBinding; -import org.springframework.context.annotation.Configuration; - -@Data -@Configuration -@ConfigurationProperties(prefix = "auth") -public class AuthProperties { - - String jwtPublicKeyUrl; - - String jwtPublicKeyStr; - - GraphqlScopes graphqlScopes; - - @Value - @ConstructorBinding - public static class GraphqlScopes { - ImmutableList queryOnly; - ImmutableList queryAndMutation; - - public GraphqlScopes(List queryOnly, List queryAndMutation) { - this.queryOnly = ImmutableList.copyOf(queryOnly); - this.queryAndMutation = ImmutableList.copyOf(queryAndMutation); - } - } -} diff --git a/src/main/java/org/icgc/argo/workflow_management/gatekeeper/graphql/model/GqlSearchQueryArgs.java b/src/main/java/org/icgc/argo/workflow_management/gatekeeper/graphql/model/GqlSearchQueryArgs.java index eef33e0..ecfc9aa 100644 --- a/src/main/java/org/icgc/argo/workflow_management/gatekeeper/graphql/model/GqlSearchQueryArgs.java +++ b/src/main/java/org/icgc/argo/workflow_management/gatekeeper/graphql/model/GqlSearchQueryArgs.java @@ -18,11 +18,11 @@ package org.icgc.argo.workflow_management.gatekeeper.graphql.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.util.List; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.icgc.argo.workflow_management.gatekeeper.model.Run; @Data diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/model/WorkflowEngineParams.java b/src/main/java/org/icgc/argo/workflow_management/wes/model/WorkflowEngineParams.java index da0a3bd..4232b4b 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/model/WorkflowEngineParams.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/model/WorkflowEngineParams.java @@ -18,6 +18,7 @@ package org.icgc.argo.workflow_management.wes.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.databind.PropertyNamingStrategy; import com.fasterxml.jackson.databind.annotation.JsonNaming; import java.util.UUID; @@ -25,7 +26,6 @@ import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; @Data @Builder diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e10c223..158f989 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -21,17 +21,6 @@ logging.level: # org.springframework: debug org.icgc.argo: debug ---- -spring.profiles: secure -auth: - jwtPublicKeyUrl: "" - jwtPublicKeyStr: "-----BEGIN PUBLIC KEY-----\nSET ME IF YOU DONT HAVE A URL, BUT URL TAKES PRIORITY\n-----END PUBLIC KEY-----" - graphqlScopes: - queryOnly: - - RDPC-CA.READ - queryAndMutation: - - RDPC-CA.WRITE - --- spring.profiles: apiKey From 3046149d9250ef512d271e4b69e610eb31ac7249 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 30 Aug 2021 11:41:09 -0400 Subject: [PATCH 21/26] nextflow config updated for plugins mode and image pull policy --- .../wes/NextflowService.java | 21 +++++++----- .../model}/NextflowConfigFile.java | 33 ++++++++++++------- .../wes/properties/NextflowProperties.java | 1 + src/main/resources/application.yml | 1 + 4 files changed, 35 insertions(+), 21 deletions(-) rename src/main/java/org/icgc/argo/workflow_management/{util => wes/model}/NextflowConfigFile.java (80%) diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java index 8f50891..da4298c 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java @@ -20,7 +20,6 @@ import static java.lang.String.format; import static java.util.Objects.nonNull; -import static org.icgc.argo.workflow_management.util.NextflowConfigFile.createNextflowConfigFile; import static org.icgc.argo.workflow_management.util.ParamsFile.createParamsFile; import static org.icgc.argo.workflow_management.util.Reflections.createWithReflection; import static org.icgc.argo.workflow_management.util.Reflections.invokeDeclaredMethod; @@ -296,14 +295,18 @@ private CmdKubeRun createCmd(@NonNull Launcher launcher, @NonNull RunParams para // Write config file for run using required and optional arguments // Use launchDir, projectDir and/or workDir if provided in workflow_engine_options val config = - createNextflowConfigFile( - runName, - k8sConfig.getRunAsUser(), - k8sConfig.getServiceAccount(), - k8sConfig.getRunNamespace(), - workflowEngineParams.getLaunchDir(), - workflowEngineParams.getProjectDir(), - workflowEngineParams.getWorkDir()); + NextflowConfigFile.builder() + .runName(runName) + .runAsUser(k8sConfig.getRunAsUser()) + .serviceAccount(k8sConfig.getServiceAccount()) + .runNamespace(k8sConfig.getRunNamespace()) + .imagePullPolicy(k8sConfig.getImagePullPolicy()) + .launchDir(workflowEngineParams.getLaunchDir()) + .projectDir(workflowEngineParams.getProjectDir()) + .workDir(workflowEngineParams.getWorkDir()) + .build() + .getConfig(); + cmdParams.put("runConfig", List.of(config)); // Resume workflow by name/id diff --git a/src/main/java/org/icgc/argo/workflow_management/util/NextflowConfigFile.java b/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java similarity index 80% rename from src/main/java/org/icgc/argo/workflow_management/util/NextflowConfigFile.java rename to src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java index 108932e..29e608b 100644 --- a/src/main/java/org/icgc/argo/workflow_management/util/NextflowConfigFile.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java @@ -16,7 +16,7 @@ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -package org.icgc.argo.workflow_management.util; +package org.icgc.argo.workflow_management.wes.model; import static com.google.common.base.Strings.isNullOrEmpty; @@ -26,20 +26,23 @@ import java.util.ArrayList; import java.util.List; import java.util.function.Consumer; +import lombok.Builder; import lombok.NonNull; import lombok.val; +@Builder public class NextflowConfigFile { - public static String createNextflowConfigFile( - @NonNull String filename, - @NonNull Integer runAsUser, - String serviceAccount, - String runNamespace, - String launchDir, - String projectDir, - String workDir) - throws IOException { - val filePath = String.format("/tmp/%s.config", filename); + private @NonNull String runName; + private @NonNull Integer runAsUser; + private String serviceAccount; + private String runNamespace; + private String imagePullPolicy; + private String launchDir; + private String projectDir; + private String workDir; + + public String getConfig() throws IOException { + val filePath = String.format("/tmp/%s.config", runName); File configFile = new File(filePath); FileWriter writer = new FileWriter(configFile); @@ -56,12 +59,18 @@ public static String createNextflowConfigFile( writeFormattedLineIfValue(fileContent::add, "\tnamespace= '%s'", runNamespace); writeFormattedLineIfValue(fileContent::add, "\tserviceAccount = '%s'", serviceAccount); + // k8s image pull policy for run + fileContent.add(String.format("\tpullPolicy = %s", imagePullPolicy)); + // variable config passed in via WorkflowEngineParams writeFormattedLineIfValue(fileContent::add, "\tlaunchDir = '%s'", launchDir); writeFormattedLineIfValue(fileContent::add, "\tprojectDir = '%s'", projectDir); writeFormattedLineIfValue(fileContent::add, "\tworkDir = '%s'", workDir); - // close it off + // set plugins mode to prod + fileContent.add("\tpod = [env: 'NXF_PLUGINS_MODE', value: 'prod']"); + + // close k8s off fileContent.add("}"); // Write contents to file diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index a17093f..9c1f75b 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -41,6 +41,7 @@ public static class K8sProperties { private String serviceAccount; private String namespace; private String runNamespace; + private String imagePullPolicy; private List volMounts; private String masterUrl; private boolean trustCertificate; diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 158f989..b381ee3 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -3,6 +3,7 @@ nextflow: runAsUser: 9999 namespace: "default" # Namespace where app will be running runNamespace: "default" # Namespace where wes and nf pods will live + imagePullPolicy: "IfNotPresent" # how K8s should pull a pod image for runs serviceAccount: "default" volMounts: - "pv-claim:/some/dir" From abc2c50e2c87e5dfffd1c73b282c913e12d8b5d6 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 30 Aug 2021 11:56:35 -0400 Subject: [PATCH 22/26] bugfix: string quotes for config --- .../argo/workflow_management/wes/model/NextflowConfigFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java b/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java index 29e608b..e40a88d 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java @@ -60,7 +60,7 @@ public String getConfig() throws IOException { writeFormattedLineIfValue(fileContent::add, "\tserviceAccount = '%s'", serviceAccount); // k8s image pull policy for run - fileContent.add(String.format("\tpullPolicy = %s", imagePullPolicy)); + fileContent.add(String.format("\tpullPolicy = '%s'", imagePullPolicy)); // variable config passed in via WorkflowEngineParams writeFormattedLineIfValue(fileContent::add, "\tlaunchDir = '%s'", launchDir); From 79f034b00cd8480b09bd2d50b5a8149cb66c1536 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 30 Aug 2021 12:24:36 -0400 Subject: [PATCH 23/26] configurable plugins dir --- .../argo/workflow_management/wes/NextflowService.java | 1 + .../workflow_management/wes/model/NextflowConfigFile.java | 8 ++++++-- .../wes/properties/NextflowProperties.java | 1 + src/main/resources/application.yml | 1 + 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java index da4298c..5f7cb94 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowService.java @@ -301,6 +301,7 @@ private CmdKubeRun createCmd(@NonNull Launcher launcher, @NonNull RunParams para .serviceAccount(k8sConfig.getServiceAccount()) .runNamespace(k8sConfig.getRunNamespace()) .imagePullPolicy(k8sConfig.getImagePullPolicy()) + .pluginsDir(k8sConfig.getPluginsDir()) .launchDir(workflowEngineParams.getLaunchDir()) .projectDir(workflowEngineParams.getProjectDir()) .workDir(workflowEngineParams.getWorkDir()) diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java b/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java index e40a88d..35a2c7c 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/model/NextflowConfigFile.java @@ -37,6 +37,7 @@ public class NextflowConfigFile { private String serviceAccount; private String runNamespace; private String imagePullPolicy; + private String pluginsDir; private String launchDir; private String projectDir; private String workDir; @@ -67,8 +68,11 @@ public String getConfig() throws IOException { writeFormattedLineIfValue(fileContent::add, "\tprojectDir = '%s'", projectDir); writeFormattedLineIfValue(fileContent::add, "\tworkDir = '%s'", workDir); - // set plugins mode to prod - fileContent.add("\tpod = [env: 'NXF_PLUGINS_MODE', value: 'prod']"); + // plugins + fileContent.add( + String.format( + "\tpod = [ [env: 'NXF_PLUGINS_MODE', value: 'prod'], [env: 'NXF_PLUGINS_DIR', value: '%s'] ]", + pluginsDir)); // close k8s off fileContent.add("}"); diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index 9c1f75b..a1aa6e7 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -43,6 +43,7 @@ public static class K8sProperties { private String runNamespace; private String imagePullPolicy; private List volMounts; + private String pluginsDir; private String masterUrl; private boolean trustCertificate; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b381ee3..6c94e92 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -7,6 +7,7 @@ nextflow: serviceAccount: "default" volMounts: - "pv-claim:/some/dir" + pluginsDir: "/icgc-argo-scratch/plugins" # where nextflow should copy plugins to for a run masterUrl: "localhost:8080" trustCertificate: false weblogUrl: "http://localhost" From dd02674ded4ffff6e9ca820ec6dd8217c89ecf3c Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Mon, 30 Aug 2021 14:08:53 -0400 Subject: [PATCH 24/26] re-org properties --- .../workflow_management/wes/properties/NextflowProperties.java | 2 +- src/main/resources/application.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java index a1aa6e7..69d4e12 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/properties/NextflowProperties.java @@ -42,8 +42,8 @@ public static class K8sProperties { private String namespace; private String runNamespace; private String imagePullPolicy; - private List volMounts; private String pluginsDir; + private List volMounts; private String masterUrl; private boolean trustCertificate; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6c94e92..b97f1fa 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -4,10 +4,10 @@ nextflow: namespace: "default" # Namespace where app will be running runNamespace: "default" # Namespace where wes and nf pods will live imagePullPolicy: "IfNotPresent" # how K8s should pull a pod image for runs + pluginsDir: "/icgc-argo-scratch/plugins" # where nextflow should copy plugins to for a run serviceAccount: "default" volMounts: - "pv-claim:/some/dir" - pluginsDir: "/icgc-argo-scratch/plugins" # where nextflow should copy plugins to for a run masterUrl: "localhost:8080" trustCertificate: false weblogUrl: "http://localhost" From c0b9556596d58485cb34035e58df8b0ac8f51c62 Mon Sep 17 00:00:00 2001 From: jaserud Date: Wed, 1 Sep 2021 09:58:35 -0400 Subject: [PATCH 25/26] fix monitor infinite loop (#184) * add counter and break * fix annotation * rearrange to make smaller --- .../workflow_management/streams/DisposableManager.java | 4 ++-- .../workflow_management/wes/NextflowWorkflowMonitor.java | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/icgc/argo/workflow_management/streams/DisposableManager.java b/src/main/java/org/icgc/argo/workflow_management/streams/DisposableManager.java index dc4bc0d..3a176f7 100644 --- a/src/main/java/org/icgc/argo/workflow_management/streams/DisposableManager.java +++ b/src/main/java/org/icgc/argo/workflow_management/streams/DisposableManager.java @@ -24,10 +24,10 @@ import java.util.concurrent.Callable; import lombok.Getter; import lombok.SneakyThrows; -import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; import reactor.core.Disposable; -@Configuration +@Component public class DisposableManager { public static final String WES_CONSUMER = "WESConsumer"; public static final String GATEKEEPER_PRODUCER = "gatekeeperProducer"; diff --git a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowWorkflowMonitor.java b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowWorkflowMonitor.java index 81bdcbb..b410f05 100644 --- a/src/main/java/org/icgc/argo/workflow_management/wes/NextflowWorkflowMonitor.java +++ b/src/main/java/org/icgc/argo/workflow_management/wes/NextflowWorkflowMonitor.java @@ -48,8 +48,11 @@ public class NextflowWorkflowMonitor implements Runnable { public void run() { boolean done = false; val podName = metadata.getWorkflow().getRunName(); - while (!done) { + int count = 0; + while (!done && count < 3) { + count++; try { + Thread.sleep(5000); PodResource pod = kubernetesClient.pods().withName(podName); val p = pod.get(); val log = pod.tailingLines(maxErrorLogLines).getLog(); @@ -58,6 +61,9 @@ public void run() { log.error(format("Workflow Status Monitor threw exception %s", e.getMessage())); } } + if (count == 3) { + log.debug("Not sure if WF is running, but I can't keep looking forever!"); + } } public boolean handlePod(NextflowMetadata metadata, Pod pod, String podLog) { From 0deeb4a1143d926b1a72d31e773a093e4af47092 Mon Sep 17 00:00:00 2001 From: Alex Lepsa Date: Thu, 2 Sep 2021 15:14:37 -0400 Subject: [PATCH 26/26] release 3.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5975f23..14de39c 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.icgc.argo workflow-management - 3.2.0-SNAPSHOT + 3.2.0 workflow-management ARGO Workflow Management