Merge branch 'develop' into update/netty-all-4.1.114.Final

.git-blame-ignore-revs

@@ -1,2 +1,5 @@
 # Scala Steward: Reformat with scalafmt 3.6.1
 0c093c52c09d7b12cdbd38008e0bbc58c9d110be
+
+# Scala Steward: Reformat with scalafmt 3.8.3
+fc6844bda9d3bdf0b5751381ed9e402fdeb577b8

.github/CODEOWNERS

@@ -0,0 +1 @@
+*   @broadinstitute/dsp-core-services

.github/workflows/auto-approve-broadbot-prs.yml

@@ -0,0 +1,23 @@
+name: Broadbot auto-approve
+on: pull_request
+
+permissions:
+  pull-requests: write
+
+jobs:
+  getActor:
+    runs-on: ubuntu-latest
+    steps:
+    - name: "Echo github actor"
+      env:
+        GH_ACTOR: ${{ github.actor }}
+      run: echo "$GH_ACTOR"
+  broadbot:
+    runs-on: ubuntu-latest
+    if: github.actor == 'broadbot'
+    steps:
+      - name: Approve a PR
+        run: gh pr review --approve "$PR_URL"
+        env:
+          PR_URL: ${{ github.event.pull_request.html_url }}
+          GH_TOKEN: ${{ secrets.BROADBOT_TOKEN }}

.github/workflows/verify_consumer_pacts.yml

@@ -319,7 +319,7 @@ jobs:
                           -e JANITOR_CLIENT_CREDENTIAL_FILE_PATH="" \
                           -e JANITOR_TRACK_RESOURCE_PROJECT_ID="" \
                           -e JANITOR_TRACK_RESOURCE_TOPIC_ID="" \
-                          sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.10_7_1.10.2_2.13.15 \
+                          sbtscala/scala-sbt:eclipse-temurin-17.0.13_11_1.10.5_2.13.15 \
                           bash -c "git config --global --add safe.directory /working/sam && sbt \"set scalafmtOnCompile := false\" \"project pact4s\" \"testOnly *SamProviderSpec\""
 
   can-i-deploy: # The can-i-deploy job will run as a result of a Sam PR. It reports the pact verification statuses on all deployed environments.

.scalafmt.conf

@@ -1,4 +1,4 @@
-version=3.6.1
+version=3.8.3
 style = default
 runner.dialect = scala213
 

automation/Dockerfile-tests

@@ -1,4 +1,4 @@
-FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.10_7_1.10.2_2.13.15
+FROM sbtscala/scala-sbt:eclipse-temurin-17.0.13_11_1.10.5_2.13.15
 
 COPY src /app/src
 COPY test.sh /app

automation/project/Dependencies.scala

@@ -29,13 +29,15 @@ object Dependencies {
     "com.fasterxml.jackson.module" % ("jackson-module-scala_" + scalaV) % jacksonV,
     "ch.qos.logback" % "logback-classic" % "1.4.5",
     "org.slf4j" % "slf4j-api" % "2.0.3",
-    "net.logstash.logback" % "logstash-logback-encoder" % "6.6",
+    "net.logstash.logback" % "logstash-logback-encoder" % "8.0",
     "com.google.apis" % "google-api-services-oauth2" % "v1-rev112-1.20.0" excludeAll (
       ExclusionRule("com.google.guava", "guava-jdk5"),
       ExclusionRule("org.apache.httpcomponents", "httpclient")
     ),
-    "com.google.api-client" % "google-api-client" % "1.22.0" excludeAll (ExclusionRule("com.google.guava", "guava-jdk5"),
-    ExclusionRule("org.apache.httpcomponents", "httpclient")),
+    "com.google.api-client" % "google-api-client" % "1.22.0" excludeAll (
+      ExclusionRule("com.google.guava", "guava-jdk5"),
+      ExclusionRule("org.apache.httpcomponents", "httpclient")
+    ),
     "com.typesafe.akka" %% "akka-http-core" % akkaHttpV,
     "com.typesafe.akka" %% "akka-stream-testkit" % akkaV,
     "com.typesafe.akka" %% "akka-http" % akkaHttpV,

automation/project/build.properties

@@ -1 +1 @@
-sbt.version=1.10.2
+sbt.version=1.10.5

codegen_java/project/build.properties

@@ -1 +1 @@
-sbt.version=1.10.2
+sbt.version=1.10.5

docker/build.sh

@@ -96,9 +96,9 @@ function make_jar()
     GIT_MODEL_HASH=$(git log -n 1 --pretty=format:%h)
 
     # make jar.  cache sbt dependencies.
-    docker run --rm --link postgres:postgres -e GIT_MODEL_HASH=$GIT_MODEL_HASH -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.10_7_1.10.2_2.13.15 /working/docker/init_schema.sh /working
+    docker run --rm --link postgres:postgres -e GIT_MODEL_HASH=$GIT_MODEL_HASH -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-17.0.13_11_1.10.5_2.13.15 /working/docker/init_schema.sh /working
     sleep 40
-    docker run --rm --link postgres:postgres -e GIT_MODEL_HASH=$GIT_MODEL_HASH -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.10_7_1.10.2_2.13.15 /working/docker/install.sh /working
+    docker run --rm --link postgres:postgres -e GIT_MODEL_HASH=$GIT_MODEL_HASH -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-17.0.13_11_1.10.5_2.13.15 /working/docker/install.sh /working
     EXIT_CODE=$?
     set -e # Turn error detection back on for the rest of the script
 

docker/build_jar.sh

@@ -8,7 +8,7 @@ set -e
 # Get the last commit hash of the model directory and set it as an environment variable
 GIT_MODEL_HASH=$(git log -n 1 --pretty=format:%h)
 
-docker run --rm -e GIT_MODEL_HASH=$GIT_MODEL_HASH -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.10_7_1.10.2_2.13.15 /working/docker/clean_install.sh /working
+docker run --rm -e GIT_MODEL_HASH=$GIT_MODEL_HASH -v $PWD:/working -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 sbtscala/scala-sbt:eclipse-temurin-17.0.13_11_1.10.5_2.13.15 /working/docker/clean_install.sh /working
 EXIT_CODE=$?
 
 if [ $EXIT_CODE != 0 ]; then

pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala

@@ -158,7 +158,7 @@ object MockTestSupport extends MockTestSupport {
       policyDAO,
       googleExt,
       FakeOpenIDConnectConfiguration,
-      azureService:Option[AzureService]
+      azureService: Option[AzureService]
     )
   }
 

project/Dependencies.scala

@@ -8,10 +8,10 @@ object Dependencies {
   val scalaTestV = "3.2.19"
   val scalaCheckV = "1.18.1"
   val scalikejdbcVersion = "3.4.2"
-  val postgresDriverVersion = "42.7.2"
+  val postgresDriverVersion = "42.7.4"
   val sentryVersion = "6.15.0"
 
-  val workbenchLibV = "80e4b8d" // If updating this, make sure googleStorageLocal in test dependencies is up-to-date
+  val workbenchLibV = "fa46370" // If updating this, make sure googleStorageLocal in test dependencies is up-to-date
   val workbenchUtilV = s"0.10-$workbenchLibV"
   val workbenchUtil2V = s"0.9-$workbenchLibV"
   val workbenchModelV = s"0.20-$workbenchLibV"
@@ -46,14 +46,14 @@ object Dependencies {
   val jacksonDatabind: ModuleID = "com.fasterxml.jackson.core" % "jackson-databind" % jacksonV
   val jacksonCore: ModuleID = "com.fasterxml.jackson.core" % "jackson-core" % jacksonV
 
-  val logstashLogback: ModuleID = "net.logstash.logback" % "logstash-logback-encoder" % "6.6"
-  val logbackClassic: ModuleID = "ch.qos.logback" % "logback-classic" % "1.4.14"
+  val logstashLogback: ModuleID = "net.logstash.logback" % "logstash-logback-encoder" % "8.0"
+  val logbackClassic: ModuleID = "ch.qos.logback" % "logback-classic" % "1.5.12"
   val ravenLogback: ModuleID = "com.getsentry.raven" % "raven-logback" % "7.8.6"
   val scalaLogging: ModuleID = "com.typesafe.scala-logging" %% "scala-logging" % scalaLoggingV
   val ficus: ModuleID = "com.iheart" %% "ficus" % "1.5.2"
 //  val stackdriverLogging: ModuleID = "org.springframework.cloud" % "spring-cloud-gcp-logging" % "1.2.8.RELEASE" excludeAll(excludeSpring, excludeSpringBoot)
   val stackdriverLogging: ModuleID = "com.google.cloud" % "google-cloud-logging-logback" % "0.127.11-alpha"
-  val janino: ModuleID = "org.codehaus.janino" % "janino" % "3.1.7" // For if-else logic in logging config
+  val janino: ModuleID = "org.codehaus.janino" % "janino" % "3.1.12" // For if-else logic in logging config
 
   val akkaActor: ModuleID = "com.typesafe.akka" %% "akka-actor" % akkaV
   val akkaSlf4j: ModuleID = "com.typesafe.akka" %% "akka-slf4j" % akkaV
@@ -65,13 +65,13 @@ object Dependencies {
   val scalaCheck: ModuleID = "org.scalacheck" %% "scalacheck" % scalaCheckV % "test"
 
   val nettyAll: ModuleID = "io.netty" % "netty-all" % "4.1.114.Final"
-  val reactorNetty: ModuleID = "io.projectreactor.netty" % "reactor-netty" % "1.0.39"
+  val reactorNetty: ModuleID = "io.projectreactor.netty" % "reactor-netty" % "1.0.48"
 
   val excludIoGrpc = ExclusionRule(organization = "io.grpc", name = "grpc-core")
   val ioGrpc: ModuleID = "io.grpc" % "grpc-core" % "1.34.1"
 
   val googleOAuth2: ModuleID = "com.google.auth" % "google-auth-library-oauth2-http" % "0.18.0" excludeAll excludIoGrpc
-  val googleStorage: ModuleID = "com.google.apis" % "google-api-services-storage" % "v1-rev20220401-1.32.1" excludeAll excludIoGrpc // force this version
+  val googleStorage: ModuleID = "com.google.apis" % "google-api-services-storage" % "v1-rev20241008-2.0.0" excludeAll excludIoGrpc // force this version
 
   val monocle: ModuleID = "com.github.julien-truffaut" %% "monocle-core" % monocleVersion
   val monocleMacro: ModuleID = "com.github.julien-truffaut" %% "monocle-macro" % monocleVersion
@@ -95,20 +95,31 @@ object Dependencies {
   val excludeGoogleAutoValue = ExclusionRule(organization = "com.google.auto.value", name = "auto-value")
   val excludeBouncyCastle = ExclusionRule("org.bouncycastle")
   val workbenchGoogle2: ModuleID =
-    "org.broadinstitute.dsde.workbench" %% "workbench-google2" % workbenchGoogle2V excludeAll (excludeWorkbenchModel, excludeWorkbenchUtil, excludeGoogleAutoValue, excludeBouncyCastle)
+    "org.broadinstitute.dsde.workbench" %% "workbench-google2" % workbenchGoogle2V excludeAll (
+      excludeWorkbenchModel,
+      excludeWorkbenchUtil,
+      excludeGoogleAutoValue,
+      excludeBouncyCastle
+    )
   val workbenchNotifications: ModuleID =
     "org.broadinstitute.dsde.workbench" %% "workbench-notifications" % workbenchNotificationsV excludeAll (excludeWorkbenchGoogle, excludeWorkbenchModel)
   val workbenchGoogleTests: ModuleID =
-    "org.broadinstitute.dsde.workbench" %% "workbench-google" % workbenchGoogleV % "test" classifier "tests" excludeAll (excludeWorkbenchUtil, excludeWorkbenchModel)
+    "org.broadinstitute.dsde.workbench" %% "workbench-google" % workbenchGoogleV % "test" classifier "tests" excludeAll (
+      excludeWorkbenchUtil,
+      excludeWorkbenchModel
+    )
   val workbenchGoogle2Tests: ModuleID =
-    "org.broadinstitute.dsde.workbench" %% "workbench-google2" % workbenchGoogle2V % "test" classifier "tests" excludeAll (excludeWorkbenchUtil, excludeWorkbenchModel)
+    "org.broadinstitute.dsde.workbench" %% "workbench-google2" % workbenchGoogle2V % "test" classifier "tests" excludeAll (
+      excludeWorkbenchUtil,
+      excludeWorkbenchModel
+    )
   val googleStorageLocal: ModuleID =
-    "com.google.cloud" % "google-cloud-nio" % "0.127.25" % "test" // needed for mocking google cloud storage. Should use same version as wb-libs
+    "com.google.cloud" % "google-cloud-nio" % "0.127.26" % "test" // needed for mocking google cloud storage. Should use same version as wb-libs
 
   val liquibaseCore: ModuleID = "org.liquibase" % "liquibase-core" % "4.2.2"
 
-  val circeYAML: ModuleID = "io.circe" %% "circe-yaml" % "0.14.2"
-  val snakeYAML: ModuleID = "org.yaml" % "snakeyaml" % "1.33"
+  val circeYAML: ModuleID = "io.circe" %% "circe-yaml" % "0.16.0"
+  val snakeYAML: ModuleID = "org.yaml" % "snakeyaml" % "2.3"
 
   val scalikeCore = "org.scalikejdbc" %% "scalikejdbc" % scalikejdbcVersion
   val scalikeCoreConfig = "org.scalikejdbc" %% "scalikejdbc-config" % scalikejdbcVersion
@@ -135,9 +146,16 @@ object Dependencies {
   )
 
   val cloudResourceLib: ModuleID =
-    "bio.terra" % "terra-cloud-resource-lib" % crlVersion excludeAll (excludeGoogleServiceUsage, excludeGoogleCloudResourceManager, excludeJerseyCore, excludeJerseyMedia, excludeSLF4J, excludeAwsSdk)
+    "bio.terra" % "terra-cloud-resource-lib" % crlVersion excludeAll (
+      excludeGoogleServiceUsage,
+      excludeGoogleCloudResourceManager,
+      excludeJerseyCore,
+      excludeJerseyMedia,
+      excludeSLF4J,
+      excludeAwsSdk
+    )
   val azureManagedApplications: ModuleID =
-    "com.azure.resourcemanager" % "azure-resourcemanager-managedapplications" % "1.0.0-beta.1"
+    "com.azure.resourcemanager" % "azure-resourcemanager-managedapplications" % "1.0.0-beta.4"
 
   def excludeSpringBoot = ExclusionRule("org.springframework.boot")
   def excludeSpringAop = ExclusionRule("org.springframework.spring-aop")
@@ -169,7 +187,7 @@ object Dependencies {
   val terraCommonLib = tclExclusions("bio.terra" % "terra-common-lib" % tclVersion classifier "plain")
 
   // was included transitively before, now explicit
-  val commonsCodec: ModuleID = "commons-codec" % "commons-codec" % "1.15"
+  val commonsCodec: ModuleID = "commons-codec" % "commons-codec" % "1.17.1"
 
   val rootDependencies = Seq(
     // proactively pull in latest versions of Jackson libs, instead of relying on the versions
@@ -227,6 +245,6 @@ object Dependencies {
 
   // Needed because it looks like the dependency overrides of wb-libs doesn't propagate to the importing project...
   val rootDependencyOverrides = Seq(
-    "org.apache.commons" % "commons-compress" % "1.26.0"
+    "org.apache.commons" % "commons-compress" % "1.26.2"
   )
 }

project/build.properties

@@ -1 +1 @@
-sbt.version=1.10.2
+sbt.version=1.10.5

project/plugins.sbt

@@ -2,7 +2,7 @@ addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.0")
 
 addSbtPlugin("io.spray" % "sbt-revolver" % "0.10.0")
 
-addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.2.1")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.2.2")
 
 addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
 

src/main/resources/swagger/api-docs.yaml

@@ -199,7 +199,7 @@ paths:
     post:
       tags:
         - Admin
-      summary: Gets a list of users for a list of Sam User IDs
+      summary: Gets a list of users for a list of Sam, Azure B2C, or Google Subject IDs
       operationId: adminGetUsersByIDs
       requestBody:
         content:

src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/AccessPolicyDAO.scala

@@ -130,7 +130,11 @@ trait AccessPolicyDAO {
       samRequestContext: SamRequestContext
   ): IO[Seq[FilterResourcesResult]]
 
-  def checkPolicyGroupsInUse(resourceId: FullyQualifiedResourceId, samRequestContext: SamRequestContext): IO[List[Map[String, String]]]
+  def findPolicyGroupsInUse(
+      resourceId: FullyQualifiedResourceId,
+      samRequestContext: SamRequestContext
+  ): IO[List[(FullyQualifiedPolicyId, FullyQualifiedPolicyId)]]
+
 }
 
 sealed abstract class LoadResourceAuthDomainResult

src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresAccessPolicyDAO.scala

@@ -951,6 +951,50 @@ class PostgresAccessPolicyDAO(
     }
   }
 
+  // Return value: [(policyToUpdate, policyToRemove)]
+  override def findPolicyGroupsInUse(
+      resourceId: FullyQualifiedResourceId,
+      samRequestContext: SamRequestContext
+  ): IO[List[(FullyQualifiedPolicyId, FullyQualifiedPolicyId)]] =
+    readOnlyTransaction("findAffectedPolicyGroups", samRequestContext) { implicit session =>
+      val groupMemberTable = GroupMemberTable.syntax("group_member_table")
+      val policyTable = PolicyTable.syntax("policy_table")
+      val parentPolicyTable = PolicyTable.syntax("parent_policy_table")
+      val resourceType = ResourceTypeTable.syntax("resource_type")
+      val resourceTable = ResourceTable.syntax("resource_table")
+
+      val query = samsql"""
+      WITH resourcePolicies as (
+        SELECT ${policyTable.groupId} as childGroupId, ${policyTable.name} as policyName
+      FROM ${PolicyTable as policyTable}
+        WHERE ${policyTable.resourceId} = (${loadResourcePKSubQuery(resourceId)})
+      )
+      SELECT resourcePolicies.policyName as childPolicyName, ${parentPolicyTable.name} as parentPolicyName, ${resourceTable.name} as parentPolicyResourceName, ${resourceType.name} as parentPolicyResourceType
+from ${GroupMemberTable as groupMemberTable}
+        JOIN resourcePolicies ON ${groupMemberTable.memberGroupId} = resourcePolicies.childGroupId
+      JOIN ${PolicyTable as parentPolicyTable} ON ${parentPolicyTable.groupId} = ${groupMemberTable.groupId}
+      JOIN ${ResourceTable as resourceTable} ON ${resourceTable.id} = ${parentPolicyTable.resourceId}
+      JOIN ${ResourceTypeTable as resourceType} ON ${resourceType.id} = ${resourceTable.resourceTypeId}
+"""
+      query
+        .map { rs =>
+          val parentPolicyResourceType = rs.get[ResourceTypeName]("parentPolicyResourceType")
+          val parentPolicyResourceId = rs.get[ResourceId]("parentPolicyResourceName")
+          val parentPolciyAccessName = rs.get[AccessPolicyName]("parentPolicyName")
+          val memberPolicyAccessName = rs.get[AccessPolicyName]("childPolicyName")
+
+          val parentPolicyFullResourceId =
+            FullyQualifiedResourceId(parentPolicyResourceType, parentPolicyResourceId)
+          val parentPolicyFullId = FullyQualifiedPolicyId(parentPolicyFullResourceId, parentPolciyAccessName)
+          val memberPolicyFullId = FullyQualifiedPolicyId(resourceId, memberPolicyAccessName)
+
+          (parentPolicyFullId, memberPolicyFullId)
+        }
+        .list()
+        .apply()
+
+    }
+
   private def deleteAllResourcePolicies(resourceId: FullyQualifiedResourceId, samRequestContext: SamRequestContext)(implicit
       session: DBSession
   ): Unit = {
@@ -975,37 +1019,6 @@ class PostgresAccessPolicyDAO(
     }
   }
 
-  override def checkPolicyGroupsInUse(resourceId: FullyQualifiedResourceId, samRequestContext: SamRequestContext): IO[List[Map[String, String]]] = {
-    val g = GroupTable.syntax("g")
-    val pg = GroupTable.syntax("pg") // problematic group
-    val gm = GroupMemberTable.syntax("gm")
-    val p = PolicyTable.syntax("p")
-
-    readOnlyTransaction("checkPolicyGroupsInUse", samRequestContext) { implicit session =>
-      val problematicGroupsQuery =
-        samsql"""select ${g.result.id}, ${g.result.name}, array_agg(${pg.name}) as ${pg.resultName.name}
-                     from ${GroupTable as g}
-                     join ${GroupMemberTable as gm} on ${g.id} = ${gm.memberGroupId}
-                     join ${GroupTable as pg} on ${gm.groupId} = ${pg.id}
-                     where ${g.id} in
-                         (select distinct ${gm.result.memberGroupId}
-                          from ${GroupMemberTable as gm}
-                          join ${PolicyTable as p} on ${gm.memberGroupId} = ${p.groupId}
-                          where ${p.resourceId} = (${loadResourcePKSubQuery(resourceId)}))
-                     group by ${g.id}, ${g.name}"""
-      problematicGroupsQuery
-        .map(rs =>
-          Map(
-            "groupId" -> rs.get[GroupPK](g.resultName.id).value.toString,
-            "groupName" -> rs.get[String](g.resultName.name),
-            "still used in group(s):" -> rs.get[String](pg.resultName.name)
-          )
-        )
-        .list()
-        .apply()
-    }
-  }
-
   override def loadPolicy(resourceAndPolicyName: FullyQualifiedPolicyId, samRequestContext: SamRequestContext): IO[Option[AccessPolicy]] =
     listPolicies(resourceAndPolicyName.resource, limitOnePolicy = Option(resourceAndPolicyName.accessPolicyName), samRequestContext).map(_.headOption)
 

src/main/scala/org/broadinstitute/dsde/workbench/sam/dataAccess/PostgresDirectoryDAO.scala

@@ -430,7 +430,19 @@ class PostgresDirectoryDAO(protected val writeDbRef: DbReference, protected val
     } else {
       readOnlyTransaction("batchLoadUsers", samRequestContext) { implicit session =>
         val userTable = UserTable.syntax
-        val loadUserQuery = samsql"select ${userTable.resultAll} from ${UserTable as userTable} where ${userTable.id} in (${samUserIds})"
+        // the with clause is to keep the query size down, we only send the samUserIds once and reuse it in each unioned query
+        val loadUserQuery =
+          samsql"""
+                  with sam_user_ids (user_id) as (values ${samUserIds.map(id => samsqls"($id)")})
+                  select ${userTable.resultAll} from ${UserTable as userTable}
+                  join sam_user_ids ids on ids.user_id = ${userTable.id}
+                  union
+                  select ${userTable.resultAll} from ${UserTable as userTable}
+                  join sam_user_ids ids on ids.user_id = ${userTable.azureB2cId}
+                  union
+                  select ${userTable.resultAll} from ${UserTable as userTable}
+                  join sam_user_ids ids on ids.user_id = ${userTable.googleSubjectId}
+                  """
 
         loadUserQuery
           .map(UserTable(userTable))