diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala index ee854f10bd..fa0e95c4d2 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/Secrets.scala @@ -72,4 +72,29 @@ object Secrets { lazy val ServiceConnectionSecret: String = getSecret("service-connection-secret") lazy val ServicePrincipalClientId: String = getSecret("service-principal-clientId") + lazy val SynapseExtensionEdogPassword: String = getSecret("synapse-extension-edog-password") + lazy val SynapseExtensionEdogTenantId: String = getSecret("synapse-extension-edog-tenant-id") + lazy val SynapseExtensionEdogUxHost: String = getSecret("synapse-extension-edog-ux-host") + lazy val SynapseExtensionEdogSspHost: String = getSecret("synapse-extension-edog-ssp-host") + lazy val SynapseExtensionEdogWorkspaceId: String = getSecret("synapse-extension-edog-workspace-id") + + lazy val SynapseExtensionDailyPassword: String = getSecret("synapse-extension-daily-password") + lazy val SynapseExtensionDailyTenantId: String = getSecret("synapse-extension-daily-tenant-id") + lazy val SynapseExtensionDailyUxHost: String = getSecret("synapse-extension-daily-ux-host") + lazy val SynapseExtensionDailySspHost: String = getSecret("synapse-extension-daily-ssp-host") + lazy val SynapseExtensionDailyWorkspaceId: String = getSecret("synapse-extension-daily-workspace-id") + + lazy val SynapseExtensionDxtPassword: String = getSecret("synapse-extension-dxt-password") + lazy val SynapseExtensionDxtTenantId: String = getSecret("synapse-extension-dxt-tenant-id") + lazy val SynapseExtensionDxtUxHost: String = getSecret("synapse-extension-dxt-ux-host") + lazy val SynapseExtensionDxtSspHost: String = getSecret("synapse-extension-dxt-ssp-host") + lazy val SynapseExtensionDxtWorkspaceId: String = getSecret("synapse-extension-dxt-workspace-id") + + lazy val SynapseExtensionMsitPassword: String = getSecret("synapse-extension-msit-password") + lazy val SynapseExtensionMsitTenantId: String = getSecret("synapse-extension-msit-tenant-id") + lazy val SynapseExtensionMsitUxHost: String = getSecret("synapse-extension-msit-ux-host") + lazy val SynapseExtensionMsitSspHost: String = getSecret("synapse-extension-msit-ssp-host") + lazy val SynapseExtensionMsitWorkspaceId: String = getSecret("synapse-extension-msit-workspace-id") + + } diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala index fe3c488fd0..e08f481b9e 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/DatabricksUtilities.scala @@ -21,7 +21,6 @@ import spray.json.{JsArray, JsObject, JsValue, _} import java.io.{File, FileInputStream} import java.time.LocalDateTime import java.util.concurrent.{TimeUnit, TimeoutException} -import scala.collection.immutable.Map import scala.collection.mutable import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future, blocking} diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionUtilities.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionUtilities.scala index 48683cc413..ed8f865717 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionUtilities.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionUtilities.scala @@ -4,9 +4,8 @@ package com.microsoft.azure.synapse.ml.nbtest.SynapseExtension import com.microsoft.azure.synapse.ml.Secrets -import com.microsoft.azure.synapse.ml.Secrets.getSynapseExtensionSecret import com.microsoft.azure.synapse.ml.build.BuildInfo -import com.microsoft.azure.synapse.ml.core.env.PackageUtils.{SparkMavenPackageList, SparkMavenRepositoryList} +import com.microsoft.azure.synapse.ml.core.env.PackageUtils.SparkMavenRepositoryList import com.microsoft.azure.synapse.ml.io.http.RESTHelpers import com.microsoft.azure.synapse.ml.io.http.RESTHelpers._ import com.microsoft.azure.synapse.ml.nbtest.SharedNotebookE2ETestUtilities._ @@ -23,6 +22,7 @@ import java.time.LocalDateTime import java.time.format.DateTimeFormatter import scala.annotation.tailrec import scala.collection.JavaConverters._ +import scala.collection.immutable.HashMap import scala.concurrent.{ExecutionContext, Future, TimeoutException, blocking} object SynapseExtensionUtilities { @@ -31,40 +31,84 @@ object SynapseExtensionUtilities { object Environment extends Enumeration { type Environment = Value - val Dev, Daily, Weekly = Value + val EDog, Daily, DXT, MSIT = Value + def withNameOpt(s: String): Option[Value] = values.find(_.toString.toLowerCase == s.toLowerCase) } - lazy val TimeoutInMillis: Int = 30 * 60 * 1000 - - lazy val BaseUri: String = s"$SSPHost/metadata" - lazy val ArtifactsUri: String = s"$BaseUri/workspaces/$WorkspaceId/artifacts" - - lazy val AadAccessTokenResource: String = Secrets.AadResource - lazy val AadAccessTokenClientId: String = "1950a258-227b-4e31-a9cf-717495945fc2" - - lazy val DefaultEnvironment = Environment.Daily - lazy val SynapseEnvironment = getWorkingEnvironment(DefaultEnvironment) + object Resource extends Enumeration { + type Resource = Value + val SSPHost, WorkspaceId, UxHost, TenantId, Password, AadAccessTokenResource, Login = Value - lazy val EnvironmentString = SynapseEnvironment match { - case Environment.Dev => "dev" - case Environment.Daily => "daily" - case Environment.Weekly => "weekly" + def withNameOpt(s: String): Option[Value] = values.find(_.toString.toLowerCase == s.toLowerCase) } - lazy val SSPHost: String = getSynapseExtensionSecret(EnvironmentString, "ssp-host") - lazy val WorkspaceId: String = getSynapseExtensionSecret(EnvironmentString, "workspace-id") - lazy val UxHost: String = getSynapseExtensionSecret(EnvironmentString, "ux-host") - lazy val TenantId: String = getSynapseExtensionSecret(EnvironmentString, "tenant-id") - lazy val Password: String = getSynapseExtensionSecret(EnvironmentString, "password") - - lazy val Folder: String = s"build_${BuildInfo.version}/synapseextension/notebooks" - lazy val StorageAccount: String = "mmlsparkbuildsynapse" - lazy val StorageContainer: String = "synapse-extension" + val TimeoutInMillis: Int = 60 * 60 * 1000 + val DefaultLogin = "login.microsoftonline.com" + val PpeLogin = "login.windows-ppe.net" + + // TODO: Edog is not yet available. + // Details: 401 Unauthorized upon creating the lakehouse + lazy val EdogResources = HashMap( + Resource.SSPHost -> Secrets.SynapseExtensionEdogSspHost, + Resource.WorkspaceId -> Secrets.SynapseExtensionEdogWorkspaceId, + Resource.UxHost -> Secrets.SynapseExtensionEdogUxHost, + Resource.TenantId -> Secrets.SynapseExtensionEdogTenantId, + Resource.Password -> Secrets.SynapseExtensionEdogPassword, + Resource.AadAccessTokenResource -> "https://analysis.windows-int.net/powerbi/api", + Resource.Login -> PpeLogin) + + lazy val DailyResources = HashMap( + Resource.SSPHost -> Secrets.SynapseExtensionDailySspHost, + Resource.WorkspaceId -> Secrets.SynapseExtensionDailyWorkspaceId, + Resource.UxHost -> Secrets.SynapseExtensionDailyUxHost, + Resource.TenantId -> Secrets.SynapseExtensionDailyTenantId, + Resource.Password -> Secrets.SynapseExtensionDailyPassword, + Resource.AadAccessTokenResource -> Secrets.AadResource, + Resource.Login -> DefaultLogin) + + lazy val DxtResources = HashMap( + Resource.SSPHost -> Secrets.SynapseExtensionDxtSspHost, + Resource.WorkspaceId -> Secrets.SynapseExtensionDxtWorkspaceId, + Resource.UxHost -> Secrets.SynapseExtensionDxtUxHost, + Resource.TenantId -> Secrets.SynapseExtensionDxtTenantId, + Resource.Password -> Secrets.SynapseExtensionDxtPassword, + Resource.AadAccessTokenResource -> Secrets.AadResource, + Resource.Login -> DefaultLogin) + + // TODO: MSIT is not yet available. + // Details: We get PowerBiFeatureDisabled and a 404 upon creating the lakehouse + lazy val MsitResources = HashMap( + Resource.SSPHost -> Secrets.SynapseExtensionMsitSspHost, + Resource.WorkspaceId -> Secrets.SynapseExtensionMsitWorkspaceId, + Resource.UxHost -> Secrets.SynapseExtensionMsitUxHost, + Resource.TenantId -> Secrets.SynapseExtensionMsitTenantId, + Resource.Password -> Secrets.SynapseExtensionMsitPassword, + Resource.AadAccessTokenResource -> Secrets.AadResource, + Resource.Login -> DefaultLogin) + + val DefaultEnvironment = Environment.Daily + val ResourceMap = getResources(DefaultEnvironment) + val SSPHost: String = ResourceMap(Resource.SSPHost) + val WorkspaceId: String = ResourceMap(Resource.WorkspaceId) + val UxHost: String = ResourceMap(Resource.UxHost) + val TenantId: String = ResourceMap(Resource.TenantId) + val Password: String = ResourceMap(Resource.Password) + val AadAccessTokenResource: String = ResourceMap(Resource.AadAccessTokenResource) + val Login: String = ResourceMap(Resource.Login) + + val BaseUri: String = s"$SSPHost/metadata" + val ArtifactsUri: String = s"$BaseUri/workspaces/$WorkspaceId/artifacts?" + + val AadAccessTokenClientId: String = "1950a258-227b-4e31-a9cf-717495945fc2" + + val Folder: String = s"build_${BuildInfo.version}/synapseextension/notebooks" + val StorageAccount: String = "mmlsparkbuildsynapse" + val StorageContainer: String = "synapse-extension" lazy val AccessToken: String = getAccessToken - lazy val Platform = Secrets.Platform.toUpperCase + val Platform: String = Secrets.Platform.toUpperCase def createSJDArtifact(path: String): String = { createSJDArtifact(path, "SparkJobDefinition") @@ -73,27 +117,23 @@ object SynapseExtensionUtilities { def updateSJDArtifact(path: String, artifactId: String, storeId: String): Artifact = { val eTag = getETagFromArtifact(artifactId) val store = Secrets.ArtifactStore.capitalize - val excludes: String = "org.scala-lang:scala-reflect," + - "org.apache.spark:spark-tags_2.12," + - "org.scalatest:scalatest_2.12," + - "org.slf4j:slf4j-api" + val sparkVersion = "3.4" + val packages: String = "com.microsoft.azure:synapseml_2.12:" + BuildInfo.version val workloadPayload = s""" |"{ | 'Default${store}ArtifactId': '$storeId', | 'ExecutableFile': '$path', - | 'SparkVersion':'3.4', + | 'SparkVersion': '$sparkVersion', | 'SparkSettings': { - | 'spark.jars.packages' : '$SparkMavenPackageList', + | 'spark.jars.packages' : '$packages', | 'spark.jars.repositories' : '$SparkMavenRepositoryList', - | 'spark.jars.excludes': '$excludes', - | 'spark.dynamicAllocation.enabled': 'false', - | 'spark.yarn.user.classpath.first': 'true', | 'spark.executorEnv.IS_$Platform': 'true', - | 'spark.sql.parquet.outputwriter': - | 'org.apache.spark.sql.execution.datasources.parquet.ParquetOutputWriter', - | 'spark.sql.parquet.vorder.enabled': 'false' + | 'spark.sql.extensions': 'com.microsoft.azure.synapse.ml.predict.PredictExtension', + | 'spark.synapse.ml.predict.enabled': 'true', + | 'spark.executor.heartbeatInterval': '60s', + | 'spark.yarn.user.classpath.first': 'true', | } |}" """.stripMargin @@ -124,6 +164,7 @@ object SynapseExtensionUtilities { |} |""".stripMargin val response = postRequest(ArtifactsUri, reqBody).asJsObject().convertTo[Artifact] + println(s"Created SJD for $runName: ${getSparkJobDefinitionLink(response.objectId)}") response.objectId } @@ -278,16 +319,16 @@ object SynapseExtensionUtilities { } def getAccessToken: String = { - val createRequest = new HttpPost(s"https://login.microsoftonline.com/$TenantId/oauth2/token") + val createRequest = new HttpPost(s"https://$Login/$TenantId/oauth2/token") createRequest.setHeader("Content-Type", "application/x-www-form-urlencoded") createRequest.setEntity( new UrlEncodedFormEntity( List( - ("resource", s"$AadAccessTokenResource"), - ("client_id", s"$AadAccessTokenClientId"), + ("resource", AadAccessTokenResource), + ("client_id", AadAccessTokenClientId), ("grant_type", "password"), - ("username", s"SynapseMLE2ETestUser@$TenantId"), - ("password", s"$Password"), + ("username", s"AdminUser@$TenantId"), + ("password", Password), ("scope", "openid") ).map(p => new BasicNameValuePair(p._1, p._2)).asJava, "UTF-8") ) @@ -295,6 +336,16 @@ object SynapseExtensionUtilities { .fields("access_token").convertTo[String] } + def getResources(defaultEnv: Environment.Value): HashMap[Resource.Value, String] = { + val environment = getWorkingEnvironment(defaultEnv) + environment match { + case Environment.EDog => EdogResources + case Environment.Daily => DailyResources + case Environment.DXT => DxtResources + case Environment.MSIT => MsitResources + } + } + def getWorkingEnvironment(defaultEnv: Environment.Value): Environment.Value = { val undefined = "" val varName = "SYNAPSE_ENVIRONMENT" @@ -305,7 +356,6 @@ object SynapseExtensionUtilities { } val result = if (envValue != None) envValue.get else defaultEnv println(s"Using environment ${result.toString}") - result } } @@ -313,6 +363,7 @@ object SynapseExtensionUtilities { object SynapseJsonProtocol extends DefaultJsonProtocol { implicit object LocalDateTimeFormat extends RootJsonFormat[LocalDateTime] { def write(dt: LocalDateTime): JsValue = JsString(dt.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)) + def read(value: JsValue): LocalDateTime = LocalDateTime.parse(value.toString().replaceAll("^\"+|\"+$", ""), DateTimeFormatter.ISO_LOCAL_DATE_TIME) @@ -322,4 +373,5 @@ object SynapseJsonProtocol extends DefaultJsonProtocol { jsonFormat3(Artifact.apply) implicit val SparkJobDefinitionExecutionResponseFormat: RootJsonFormat[SparkJobDefinitionExecutionResponse] = jsonFormat3(SparkJobDefinitionExecutionResponse.apply) + } diff --git a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionsTests.scala b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionsTests.scala index 297bc13d26..f2e7a2d96f 100644 --- a/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionsTests.scala +++ b/core/src/test/scala/com/microsoft/azure/synapse/ml/nbtest/SynapseExtension/SynapseExtensionsTests.scala @@ -15,6 +15,7 @@ import scala.concurrent.{Await, Future, blocking} import scala.language.existentials class SynapseExtensionTestCleanup extends TestBase { + test("Clean up old artifacts") { SynapseExtensionUtilities.listArtifacts() .foreach(artifact => { if (artifact.lastUpdatedDate.isBefore(LocalDateTime.now().minusDays(3))) { @@ -24,6 +25,7 @@ class SynapseExtensionTestCleanup extends TestBase { SynapseExtensionUtilities.deleteArtifact(artifact.objectId) } }) + } } class SynapseExtensionsTests extends TestBase { @@ -31,21 +33,35 @@ class SynapseExtensionsTests extends TestBase { val selectedPythonFiles: Array[File] = FileUtilities.recursiveListFiles(SharedNotebookE2ETestUtilities.NotebooksDir) .filter(_.getAbsolutePath.endsWith(".py")) - .filterNot(_.getAbsolutePath.contains("EffectsOfOutreach")) - .filterNot(_.getAbsolutePath.contains("HyperParameterTuning")) - .filterNot(_.getAbsolutePath.contains("CyberML")) - .filterNot(_.getAbsolutePath.contains("VowpalWabbitOverview")) - .filterNot(_.getAbsolutePath.contains("VowpalWabbitClassificationusingVW")) - .filterNot(_.getAbsolutePath.contains("VowpalWabbitMulticlass")) - .filterNot(_.getAbsolutePath.contains("Interpretability")) //TODO: Remove when fixed - .filterNot(_.getAbsolutePath.contains("IsolationForest")) - .filterNot(_.getAbsolutePath.contains("ExplanationDashboard")) - .filterNot(_.getAbsolutePath.contains("DeepLearning")) - .filterNot(_.getAbsolutePath.contains("Cognitive")) // Excluding CogServices notebooks until GetSecret API is avail - .filterNot(_.getAbsolutePath.contains("Geospatial")) - .filterNot(_.getAbsolutePath.contains("SentimentAnalysis")) - .filterNot(_.getAbsolutePath.contains("SparkServing")) // Not testing this functionality - .filterNot(_.getAbsolutePath.contains("OpenCVPipelineImage")) // Reenable with spark streaming fix + .filterNot(_.getAbsolutePath.contains("Finetune")) // Excluded by design task 1829306 + .filterNot(_.getAbsolutePath.contains("VWnativeFormat")) + .filterNot(_.getAbsolutePath.contains("VowpalWabbitMulticlassclassification")) // Wait for Synapse fix + .filterNot(_.getAbsolutePath.contains("Langchain")) // Wait for Synapse fix + .filterNot(_.getAbsolutePath.contains("DocumentQuestionandAnsweringwithPDFs")) // Wait for Synapse fix + .filterNot(_.getAbsolutePath.contains("SetupCognitive")) // No code to run + .filterNot(_.getAbsolutePath.contains("CreateaSparkCluster")) // No code to run + .filterNot(_.getAbsolutePath.contains("Deploying")) // New issue + .filterNot(_.getAbsolutePath.contains("MultivariateAnomaly")) // New issue + .filterNot(_.getAbsolutePath.contains("TuningHyperOpt")) // New issue + .filterNot(_.getAbsolutePath.contains("IsolationForests")) // New issue + .filterNot(_.getAbsolutePath.contains("CreateAudiobooks")) // New issue + .filterNot(_.getAbsolutePath.contains("ExplanationDashboard")) // New issue + // .filterNot(_.getAbsolutePath.contains("EffectsOfOutreach")) + // .filterNot(_.getAbsolutePath.contains("HyperParameterTuning")) + // .filterNot(_.getAbsolutePath.contains("CyberML")) + // .filterNot(_.getAbsolutePath.contains("VowpalWabbitOverview")) + // .filterNot(_.getAbsolutePath.contains("VowpalWabbitClassificationusingVW")) + // .filterNot(_.getAbsolutePath.contains("VowpalWabbitMulticlass")) + // .filterNot(_.getAbsolutePath.contains("Interpretability")) //TODO: Remove when fixed + // .filterNot(_.getAbsolutePath.contains("IsolationForest")) + // .filterNot(_.getAbsolutePath.contains("ExplanationDashboard")) + // .filterNot(_.getAbsolutePath.contains("DeepLearning")) + // .filterNot(_.getAbsolutePath.contains("Cognitive")) + // Excluding CogServices notebooks until GetSecret API is avail + // .filterNot(_.getAbsolutePath.contains("Geospatial")) + // .filterNot(_.getAbsolutePath.contains("SentimentAnalysis")) + // .filterNot(_.getAbsolutePath.contains("SparkServing")) // Not testing this functionality + // .filterNot(_.getAbsolutePath.contains("OpenCVPipelineImage")) // Reenable with spark streaming fix .sortBy(_.getAbsolutePath) selectedPythonFiles.foreach(println) @@ -53,7 +69,7 @@ class SynapseExtensionsTests extends TestBase { val storeArtifactId = SynapseExtensionUtilities.createStoreArtifact() - selectedPythonFiles.seq.map(createAndExecuteSJD) + selectedPythonFiles.seq.foreach(createAndExecuteSJD) def createAndExecuteSJD(notebookFile: File): Future[String] = { val notebookName = SynapseExtensionUtilities.getBlobNameFromFilepath(notebookFile.getPath) @@ -80,4 +96,4 @@ class SynapseExtensionsTests extends TestBase { } SynapseExtensionUtilities.monitorJob(artifactId, jobInstanceId) } -} +} \ No newline at end of file diff --git a/pipeline.yaml b/pipeline.yaml index 6d6b6cc840..7af2b87b7a 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -39,20 +39,6 @@ schedules: include: - master -parameters: - - name: runSynapseExtensionE2ETests - displayName: Run Synapse Extension E2E Tests - type: boolean - default: true - - name: SYNAPSE_ENVIRONMENT - displayName: Synapse Extension E2E Test Environment - type: string - default: weekly - values: - - dev - - daily - - weekly - variables: runTests: True CONDA_CACHE_DIR: /usr/share/miniconda/envs @@ -152,9 +138,8 @@ jobs: TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.DatabricksRapidsTests" synapse: TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.SynapseTests" -# ${{ if eq(parameters.runSynapseExtensionE2ETests, true) }}: -# synapse-internal: -# TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.SynapseExtension.SynapseExtensionsTests" + synapse-internal: + TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.SynapseExtension.SynapseExtensionsTests" steps: #- template: templates/ivy_cache.yml - template: templates/update_cli.yml