From 91469f6d353128c99c15660d9f2c252b13fb1a2d Mon Sep 17 00:00:00 2001 From: Tristan Garwood Date: Wed, 25 Jan 2023 19:20:52 -0500 Subject: [PATCH] IA-3814 Pact provider test (#928) * update deps * Add pact provider tests. * Add pact provider tests. * Updated code to pull consumer contract from Pact broker. Publish verification status to Pact broker. * Rerun workflow * Rerun workflow * Attempt to fix build error. * Attempt to fix build error. * Debug library version conflicts. * Revert changes * Comment out unused dependencies for now. * Updated merge strategy * Clean up * Moved provider test to separate project. * Fixed missing import. * Updated merge strategy and settings. * revert cganges * Add logback config. * Updated logback version to 1.4.4 * revert to logback 1.2.11 * Updated logger to logback * Updated slf4j to 2.0.6 * Clean up * Add envvars for pact tests * Rerun Jenkins test * Refactored code. * Refactored code * Refactored code * Updated dependencies * Skip provider test in build.yml * Refactored code. * Refactored code. * Refactored code. * Clean up. * Rerun failed Jenkins test * Clean up. * Updated workflow trigger. Co-authored-by: Kai Co-authored-by: Ivan --- .github/workflows/verify_consumer_pacts.yml | 64 +++++ build.sbt | 5 + pact4s/src/test/resources/logback.xml | 14 + pact4s/src/test/resources/reference.conf | 198 +++++++++++++ pact4s/src/test/resources/sam.conf | 3 + .../dsde/workbench/sam/MockTestSupport.scala | 262 ++++++++++++++++++ .../sam/provider/SamProviderSpec.scala | 155 +++++++++++ project/Dependencies.scala | 18 +- project/Merging.scala | 5 + project/Settings.scala | 16 +- .../workbench/sam/api/MockSamRoutes.scala | 122 ++++++++ .../workbench/sam/api/MockStatusRoutes.scala | 42 +++ 12 files changed, 900 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/verify_consumer_pacts.yml create mode 100644 pact4s/src/test/resources/logback.xml create mode 100644 pact4s/src/test/resources/reference.conf create mode 100644 pact4s/src/test/resources/sam.conf create mode 100644 pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala create mode 100644 pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala create mode 100644 src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockSamRoutes.scala create mode 100644 src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockStatusRoutes.scala diff --git a/.github/workflows/verify_consumer_pacts.yml b/.github/workflows/verify_consumer_pacts.yml new file mode 100644 index 0000000000..9b69e04888 --- /dev/null +++ b/.github/workflows/verify_consumer_pacts.yml @@ -0,0 +1,64 @@ +name: Verify consumer pacts +# The purpose of this workflow is to verify Leo consumer contract +# using the Pact framework. +# +# The workflow requires Pact broker credentials +# - PACT_BROKER_USERNAME - the Pact Broker username +# - PACT_BROKER_PASSWORD - the Pact Broker password +on: + pull_request: + branches: + - develop + paths-ignore: + - 'README.md' + push: + branches: + - develop + paths-ignore: + - 'README.md' +env: + PACT_BROKER_URL: https://pact-broker.dsp-eng-tools.broadinstitute.org + +jobs: + verify-consumer-pact: + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + + steps: + - name: Checkout current code + uses: actions/checkout@v3 + + - name: Extract branch + id: extract-branch + run: | + GITHUB_EVENT_NAME=${{ github.event_name }} + if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then + GITHUB_REF=${{ github.ref }} + GITHUB_SHA=${{ github.sha }} + elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + GITHUB_REF=refs/heads/${{ github.head_ref }} + GITHUB_SHA=${{ github.event.pull_request.head.sha }} + else + echo "Failed to extract branch information" + exit 1 + fi + echo "ref=$GITHUB_REF" >> $GITHUB_OUTPUT + echo "sha=$GITHUB_SHA" >> $GITHUB_OUTPUT + echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "branch=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_OUTPUT + + - name: Verify consumer pacts and publish verification status to Pact Broker + run: | + docker run --rm -v $PWD:/working \ + -v jar-cache:/root/.ivy \ + -v jar-cache:/root/.ivy2 \ + -w /working \ + -e BRANCH=${{ steps.extract-branch.outputs.branch }} \ + -e GIT_SHA_SHORT=${{ steps.extract-branch.outputs.sha_short }} \ + -e PACT_BROKER_URL=${{ env.PACT_BROKER_URL }} \ + -e PACT_BROKER_USERNAME=${{ secrets.PACT_BROKER_USERNAME }} \ + -e PACT_BROKER_PASSWORD=${{ secrets.PACT_BROKER_PASSWORD }} \ + sbtscala/scala-sbt:openjdk-17.0.2_1.7.2_2.13.10 \ + sbt "project pact4s" "testOnly *SamProviderSpec" diff --git a/build.sbt b/build.sbt index 60ec4a2fbf..9476e17de5 100644 --- a/build.sbt +++ b/build.sbt @@ -6,6 +6,11 @@ lazy val root = project .settings(rootSettings: _*) .withTestSettings +lazy val pact4s = project + .in(file("pact4s")) + .settings(pact4sSettings) + .dependsOn(root % "test->test;compile->compile") + Revolver.settings Global / excludeLintKeys += debugSettings // To avoid lint warning diff --git a/pact4s/src/test/resources/logback.xml b/pact4s/src/test/resources/logback.xml new file mode 100644 index 0000000000..24316805e7 --- /dev/null +++ b/pact4s/src/test/resources/logback.xml @@ -0,0 +1,14 @@ + + + + %d [%thread] %-5level %logger{35} - %msg%n + + + + + + + + + + diff --git a/pact4s/src/test/resources/reference.conf b/pact4s/src/test/resources/reference.conf new file mode 100644 index 0000000000..703b17a463 --- /dev/null +++ b/pact4s/src/test/resources/reference.conf @@ -0,0 +1,198 @@ +googleServices { + appName = "firecloud:sam" + appsDomain = "dev.test.firecloud.org" + environment = "local" + pathToPem = "/etc/sam-account.pem" + pathToDefaultCredentialJson = "fakePath" + serviceAccountClientId = "109949113883754608360" + serviceAccountClientEmail = "sam-dev-service-account@broad-dsde-dev.iam.gserviceaccount.com" + serviceAccountClientProject = "broad-dsde-dev" + subEmail = "google@dev.test.firecloud.org" + projectServiceAccount = "broad-dsde-dev@gs-project-accounts.iam.gserviceaccount.com" + terraGoogleOrgNumber = "mock-org-number" # This org number needs to match what is specified in workbench-libs/google/src/main/scala/org/broadinstitute/dsde/workbench/google/HttpGoogleProjectDAO.scala.getAncestry + + groupSync { + pubSubProject = proj + pollInterval = 10ms + pollJitter = 0s + pubSubTopic = top + pubSubSubscription = sub + workerCount = 1 + } + + disableUsers { + pubSubProject = proj + pollInterval = 10ms + pollJitter = 0s + pubSubTopic = top + pubSubSubscription = sub + workerCount = 1 + } + + notifications { + project = proj + topicName = "notifications" + } + + googleKeyCache { + bucketName = "my-test-bucket" + activeKeyMaxAge = 25000 #test objects default to Jan 1 1970. Cranking this value up allows for test keys to be seen as active + retiredKeyMaxAge = 25048 + + monitor { + pubSubProject = "broad-dsde-dev" + pollInterval = 1m + pollJitter = 10s + pubSubTopic = "sam-google-key-cache" + pubSubSubscription = "sam-google-key-cache-sub" + workerCount = 1 + } + } + + kms { + project = "broad-dsde-dev" + location = "global" + keyRingId = "not-actually-used" + keyId = "dockerhub-key" + rotationPeriod = "180 days" + } +} + +db { + enabled=false +} + +termsOfService { + enabled = false + version = 1 + url = "app.terra.bio/#terms-of-service" +} + +petServiceAccount { + googleProject = "my-pet-project" + serviceAccountUsers = ["some-other-sa@test.iam.gserviceaccount.com"] +} + +testStuff = { + resourceTypes = { + testType = { + actionPatterns = { + alter_policies = { + description = "" + authDomainConstrainable = true + } + read_policies = { + description = "" + } + } + ownerRoleName = "owner" + roles = { + owner = { + roleActions = ["alter_policies", "read_policies"], + includedRoles = ["nonOwner"], + descendantRoles = { + otherType = ["owner"] + } + }, + nonOwner = { + roleActions = [] + } + } + reuseIds = false + } + } + # Test MRG in which to create managed identities + azure { + tenantId = "0cb7a640-45a2-4ed6-be9f-63519f86e04b" + subscriptionId = "3efc5bdf-be0e-44e7-b1d7-c08931e3c16c" + managedResourceGroupName = "mrg-terra-dev-previ-20220930130036" + } +} + +// dummy value for testing only +oidc { + authorityEndpoint = "https://accounts.google.com" + oidcClientId = "some-client" + oidcClientSecret = "some-secret" + legacyGoogleClientId = "another-client" +} + +liquibase { + changelog = "org/broadinstitute/dsde/sam/liquibase/changelog.xml" + initWithLiquibase = true +} + +db { + enabled = true + enabled = ${?postgres.enabled} + + sam_read { + poolName = "sam_read" + poolInitialSize = 5 + poolMaxSize = 5 + poolConnectionTimeoutMillis = 5000 + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://localhost:5432/testdb?stringtype=unspecified" + user = "sam-test" + password = "sam-test" + } + + sam_write { + poolName = "sam_write" + poolInitialSize = 5 + poolMaxSize = 5 + poolConnectionTimeoutMillis = 5000 + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://localhost:5432/testdb?stringtype=unspecified" + user = "sam-test" + password = "sam-test" + } + + // this background pool is used to test the status of a pool that cannot connect + sam_background { + poolName = "sam_background" + poolInitialSize = 5 + poolMaxSize = 5 + poolConnectionTimeoutMillis = 5000 + driver = "org.postgresql.Driver" + url = "jdbc:postgresql://does_not_exist/testdb?stringtype=unspecified" + user = "sam-test" + password = "sam-test" + } +} + +scalikejdbc.global.loggingSQLAndTime { + enabled = false, # switch this to true to print sql + singleLineMode = true, # switch this to false to see stack trace information about where sql was executed + printUnprocessedStackTrace = false, + stackTraceDepth= 15, + logLevel = warn, # this is set to warn so we don't have to fiddle with logback settings too + warningEnabled = false, + warningThresholdMillis = 3000, + warningLogLevel = warn +} + + +dataStore { + live = postgres +} + +termsOfService { + enabled = false + isGracePeriodEnabled = false + version = 0 + url = "app.terra.bio/#terms-of-service" +} + +admin { + superAdminsGroup = "sam-super-admins@dev.test.firecloud.org" + allowedAdminEmailDomains = ["test.firecloud.org"] +} + +prometheus { + endpointPort = 0 +} + +oidc { + oidcClientId = "0" +} \ No newline at end of file diff --git a/pact4s/src/test/resources/sam.conf b/pact4s/src/test/resources/sam.conf new file mode 100644 index 0000000000..0c9b3f75c6 --- /dev/null +++ b/pact4s/src/test/resources/sam.conf @@ -0,0 +1,3 @@ +prometheus { + endpointPort = 1 +} diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala new file mode 100644 index 0000000000..73a962a300 --- /dev/null +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/MockTestSupport.scala @@ -0,0 +1,262 @@ +package org.broadinstitute.dsde.workbench.sam + +import akka.actor.ActorSystem +import akka.stream.Materializer +import cats.effect._ +import cats.effect.unsafe.implicits.global +import cats.kernel.Eq +import com.typesafe.config.ConfigFactory +import net.ceedubs.ficus.Ficus._ +import org.broadinstitute.dsde.workbench.dataaccess.PubSubNotificationDAO +import org.broadinstitute.dsde.workbench.google.mock._ +import org.broadinstitute.dsde.workbench.google.{GoogleDirectoryDAO, GoogleIamDAO} +import org.broadinstitute.dsde.workbench.google2.mock.FakeGoogleStorageInterpreter +import org.broadinstitute.dsde.workbench.model._ +import org.broadinstitute.dsde.workbench.oauth2.OpenIDConnectConfiguration +import org.broadinstitute.dsde.workbench.oauth2.mock.FakeOpenIDConnectConfiguration +import org.broadinstitute.dsde.workbench.openTelemetry.{FakeOpenTelemetryMetricsInterpreter, OpenTelemetryMetrics, OpenTelemetryMetricsInterpreter} +import org.broadinstitute.dsde.workbench.sam.api._ +import org.broadinstitute.dsde.workbench.sam.azure.{AzureService, MockCrlService} +import org.broadinstitute.dsde.workbench.sam.config.AppConfig._ +import org.broadinstitute.dsde.workbench.sam.config._ +import org.broadinstitute.dsde.workbench.sam.dataAccess._ +import org.broadinstitute.dsde.workbench.sam.db.TestDbReference +import org.broadinstitute.dsde.workbench.sam.db.tables._ +import org.broadinstitute.dsde.workbench.sam.google.{GoogleExtensionRoutes, GoogleExtensions, GoogleGroupSynchronizer, GoogleKeyCache} +import org.broadinstitute.dsde.workbench.sam.model._ +import org.broadinstitute.dsde.workbench.sam.service.UserService._ +import org.broadinstitute.dsde.workbench.sam.service._ +import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext +import org.scalatest.Tag +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.time.{Seconds, Span} +import scalikejdbc.QueryDSL.delete +import scalikejdbc.withSQL + +import java.util.concurrent.Executors +import scala.concurrent.ExecutionContext.Implicits.{global => globalEc} +import scala.concurrent.duration._ +import scala.concurrent.{Await, Awaitable, ExecutionContext} + +trait MockTestSupport { + def runAndWait[T](f: Awaitable[T]): T = Await.result(f, Duration.Inf) + def runAndWait[T](f: IO[T]): T = f.unsafeRunSync() + + implicit val futureTimeout = Timeout(Span(10, Seconds)) + implicit val eqWorkbenchException: Eq[WorkbenchException] = (x: WorkbenchException, y: WorkbenchException) => x.getMessage == y.getMessage + implicit val openTelemetry = FakeOpenTelemetryMetricsInterpreter + + val samRequestContext = SamRequestContext() + + def dummyResourceType(name: ResourceTypeName) = + ResourceType(name, Set.empty, Set(ResourceRole(ResourceRoleName("owner"), Set.empty)), ResourceRoleName("owner")) +} + +object MockTestSupport extends MockTestSupport { + private val executor = Executors.newCachedThreadPool() + val blockingEc = ExecutionContext.fromExecutor(executor) + + val config = ConfigFactory.load() + val appConfig = AppConfig.readConfig(config) + val petServiceAccountConfig = appConfig.googleConfig.get.petServiceAccountConfig + val googleServicesConfig = appConfig.googleConfig.get.googleServicesConfig + val configResourceTypes = config.as[Map[String, ResourceType]]("resourceTypes").values.map(rt => rt.name -> rt).toMap + val adminConfig = config.as[AdminConfig]("admin") + val databaseEnabled = config.getBoolean("db.enabled") + val databaseEnabledClue = "-- skipping tests that talk to a real database" + + lazy val distributedLock = PostgresDistributedLockDAO[IO](dbRef, dbRef, appConfig.distributedLockConfig) + def proxyEmail(workbenchUserId: WorkbenchUserId) = WorkbenchEmail(s"PROXY_$workbenchUserId@${googleServicesConfig.appsDomain}") + def genGoogleSubjectId(): Option[GoogleSubjectId] = Option(GoogleSubjectId(genRandom(System.currentTimeMillis()))) + def genAzureB2CId(): AzureB2CId = AzureB2CId(genRandom(System.currentTimeMillis())) + + def genSamDependencies( + resourceTypes: Map[ResourceTypeName, ResourceType] = Map.empty, + googIamDAO: Option[GoogleIamDAO] = None, + googleServicesConfig: GoogleServicesConfig = googleServicesConfig, + cloudExtensions: Option[CloudExtensions] = None, + googleDirectoryDAO: Option[GoogleDirectoryDAO] = None, + policyAccessDAO: Option[AccessPolicyDAO] = None, + policyEvaluatorServiceOpt: Option[PolicyEvaluatorService] = None, + resourceServiceOpt: Option[ResourceService] = None, + tosEnabled: Boolean = false + )(implicit system: ActorSystem) = { + val googleDirectoryDAO = new MockGoogleDirectoryDAO() + val directoryDAO = new MockDirectoryDAO() + val googleIamDAO = googIamDAO.getOrElse(new MockGoogleIamDAO()) + val policyDAO = policyAccessDAO.getOrElse(new MockAccessPolicyDAO(resourceTypes, directoryDAO)) + val notificationPubSubDAO = new MockGooglePubSubDAO() + val googleGroupSyncPubSubDAO = new MockGooglePubSubDAO() + val googleDisableUsersPubSubDAO = new MockGooglePubSubDAO() + val googleKeyCachePubSubDAO = new MockGooglePubSubDAO() + val googleStorageDAO = new MockGoogleStorageDAO() + val googleProjectDAO = new MockGoogleProjectDAO() + val notificationDAO = new PubSubNotificationDAO(notificationPubSubDAO, "foo") + val cloudKeyCache = new GoogleKeyCache( + distributedLock, + googleIamDAO, + googleStorageDAO, + FakeGoogleStorageInterpreter, + googleKeyCachePubSubDAO, + googleServicesConfig, + petServiceAccountConfig + ) + val googleExt = cloudExtensions.getOrElse( + new GoogleExtensions( + distributedLock, + directoryDAO, + policyDAO, + googleDirectoryDAO, + notificationPubSubDAO, + googleGroupSyncPubSubDAO, + googleDisableUsersPubSubDAO, + googleIamDAO, + googleStorageDAO, + googleProjectDAO, + cloudKeyCache, + notificationDAO, + FakeGoogleKmsInterpreter, + googleServicesConfig, + petServiceAccountConfig, + resourceTypes, + adminConfig.superAdminsGroup + ) + ) + val policyEvaluatorService = policyEvaluatorServiceOpt.getOrElse(PolicyEvaluatorService(appConfig.emailDomain, resourceTypes, policyDAO, directoryDAO)) + val mockResourceService = resourceServiceOpt.getOrElse( + new ResourceService( + resourceTypes, + policyEvaluatorService, + policyDAO, + directoryDAO, + googleExt, + emailDomain = "example.com", + adminConfig.allowedEmailDomains + ) + ) + val mockManagedGroupService = + new ManagedGroupService(mockResourceService, policyEvaluatorService, resourceTypes, policyDAO, directoryDAO, googleExt, "example.com") + val tosService = new TosService(directoryDAO, googleServicesConfig.appsDomain, tosConfig.copy(enabled = tosEnabled)) + val azureService = new AzureService(MockCrlService(), directoryDAO, new MockAzureManagedResourceGroupDAO) + MockSamDependencies( + mockResourceService, + policyEvaluatorService, + tosService, + new UserService(directoryDAO, googleExt, Seq.empty, tosService), + new StatusService(directoryDAO, googleExt, dbRef), + mockManagedGroupService, + directoryDAO, + policyDAO, + googleExt, + FakeOpenIDConnectConfiguration, + azureService + ) + } + + val tosConfig = config.as[TermsOfServiceConfig]("termsOfService") + + def genSamRoutes(samDependencies: MockSamDependencies, uInfo: SamUser)(implicit + system: ActorSystem, + materializer: Materializer, + openTelemetry: OpenTelemetryMetrics[IO] + ): MockSamRoutes = new MockSamRoutes( + samDependencies.resourceService, + samDependencies.userService, + samDependencies.statusService, + samDependencies.managedGroupService, + samDependencies.tosService.tosConfig, + samDependencies.directoryDAO, + samDependencies.policyEvaluatorService, + samDependencies.tosService, + LiquibaseConfig("", false), + samDependencies.oauth2Config, + Some(samDependencies.azureService) + ) with MockSamUserDirectives with GoogleExtensionRoutes { + override val cloudExtensions: CloudExtensions = samDependencies.cloudExtensions + override val googleExtensions: GoogleExtensions = samDependencies.cloudExtensions match { + case extensions: GoogleExtensions => extensions + case _ => null + } + override val googleGroupSynchronizer: GoogleGroupSynchronizer = + if (samDependencies.cloudExtensions.isInstanceOf[GoogleExtensions]) { + new GoogleGroupSynchronizer( + googleExtensions.directoryDAO, + googleExtensions.accessPolicyDAO, + googleExtensions.googleDirectoryDAO, + googleExtensions, + googleExtensions.resourceTypes + )(executionContext) + } else null + val googleKeyCache = samDependencies.cloudExtensions match { + case extensions: GoogleExtensions => extensions.googleKeyCache + case _ => null + } + override val user: SamUser = uInfo + override val newSamUser: Option[SamUser] = Option(uInfo) + } + + def genSamRoutesWithDefault(implicit system: ActorSystem, materializer: Materializer, openTelemetry: OpenTelemetryMetricsInterpreter[IO]): MockSamRoutes = + genSamRoutes(genSamDependencies(), Generator.genWorkbenchUserBoth.sample.get) + + /* +In unit tests there really is not a difference between read and write pools. +Ideally I would not even have it. But I also want to have DatabaseNames enum and DbReference.init to use it. +So the situation is a little messy and I favor having more mess on the test side than the production side +(i.e. I don't want to add a new database name just for tests). +So, just use the DatabaseNames.Read connection pool for tests. + */ + lazy val dbRef = TestDbReference.init(config.as[LiquibaseConfig]("liquibase"), appConfig.samDatabaseConfig.samRead.dbName, MockTestSupport.blockingEc) + + def truncateAll: Int = + if (databaseEnabled) { + dbRef.inLocalTransaction { implicit session => + val tables = List( + PolicyActionTable, + PolicyRoleTable, + PolicyTable, + AuthDomainTable, + ResourceTable, + RoleActionTable, + ResourceActionTable, + NestedRoleTable, + ResourceRoleTable, + ResourceActionPatternTable, + ResourceTypeTable, + GroupMemberTable, + GroupMemberFlatTable, + PetServiceAccountTable, + AzureManagedResourceGroupTable, + PetManagedIdentityTable, + UserTable, + AccessInstructionsTable, + GroupTable + ) + + tables + .map(table => + withSQL { + delete.from(table) + }.update().apply() + ) + .sum + } + } else { + 0 + } +} + +final case class MockSamDependencies( + resourceService: ResourceService, + policyEvaluatorService: PolicyEvaluatorService, + tosService: TosService, + userService: UserService, + statusService: StatusService, + managedGroupService: ManagedGroupService, + directoryDAO: DirectoryDAO, + policyDao: AccessPolicyDAO, + cloudExtensions: CloudExtensions, + oauth2Config: OpenIDConnectConfiguration, + azureService: AzureService +) + +object ConnectedTest extends Tag("connected test") diff --git a/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala new file mode 100644 index 0000000000..cd70a96543 --- /dev/null +++ b/pact4s/src/test/scala/org/broadinstitute/dsde/workbench/sam/provider/SamProviderSpec.scala @@ -0,0 +1,155 @@ +package org.broadinstitute.dsde.workbench.sam.provider + +import akka.http.scaladsl.Http +import akka.http.scaladsl.testkit.ScalatestRouteTest +import cats.effect.IO +import cats.effect.unsafe.implicits.global +import com.typesafe.scalalogging.LazyLogging +import org.broadinstitute.dsde.workbench.model.{GoogleSubjectId, WorkbenchEmail, WorkbenchUserId} +import org.broadinstitute.dsde.workbench.oauth2.mock.FakeOpenIDConnectConfiguration +import org.broadinstitute.dsde.workbench.sam.MockTestSupport.genSamRoutes +import org.broadinstitute.dsde.workbench.sam.azure.AzureService +import org.broadinstitute.dsde.workbench.sam.dataAccess.{AccessPolicyDAO, DirectoryDAO} +import org.broadinstitute.dsde.workbench.sam.google.GoogleExtensions +import org.broadinstitute.dsde.workbench.sam.model._ +import org.broadinstitute.dsde.workbench.sam.service._ +import org.broadinstitute.dsde.workbench.sam.util.SamRequestContext +import org.broadinstitute.dsde.workbench.sam.{Generator, MockSamDependencies, MockTestSupport} +import org.broadinstitute.dsde.workbench.util.health.{StatusCheckResponse, SubsystemStatus, Subsystems} +import org.mockito.ArgumentMatchers.any +import org.mockito.Mockito.when +import org.scalatest.BeforeAndAfterAll +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatestplus.mockito.MockitoSugar.mock +import pact4s.provider.Authentication.BasicAuth +import pact4s.provider._ +import pact4s.scalatest.PactVerifier + +import java.lang.Thread.sleep +import scala.concurrent.Future +import scala.concurrent.duration.DurationInt + +class SamProviderSpec extends AnyFlatSpec with ScalatestRouteTest with MockTestSupport with BeforeAndAfterAll with PactVerifier with LazyLogging { + def genSamDependencies: MockSamDependencies = { + val directoryDAO = mock[DirectoryDAO] + val policyDAO = mock[AccessPolicyDAO] + val googleExt = mock[GoogleExtensions] + + val policyEvaluatorService = mock[PolicyEvaluatorService] + val mockResourceService = mock[ResourceService] + val mockManagedGroupService = mock[ManagedGroupService] + val tosService = mock[TosService] + val azureService = mock[AzureService] + val userService = mock[UserService] + val statusService = mock[StatusService] + when { + statusService.getStatus() + } thenReturn { + Future.successful( + StatusCheckResponse( + ok = true, + Map( + Subsystems.GoogleGroups -> SubsystemStatus(ok = true, None), + Subsystems.GoogleIam -> SubsystemStatus(ok = true, None), + Subsystems.GooglePubSub -> SubsystemStatus(ok = true, None), + Subsystems.OpenDJ -> SubsystemStatus(ok = true, None) + ) + ) + ) + } + + when { + directoryDAO.loadUserByGoogleSubjectId(any[GoogleSubjectId], any[SamRequestContext]) + } thenReturn { + val samUser = SamUser(WorkbenchUserId("test"), None, WorkbenchEmail("test@test"), None, enabled = true, None) + IO.pure(Option(samUser)) + } + + when { + tosService.isTermsOfServiceStatusAcceptable(any[SamUser]) + } thenReturn true + + val fakeWorkspaceResourceType = ResourceType(ResourceTypeName("workspace"), Set.empty, Set.empty, ResourceRoleName("workspace")) + when { + mockResourceService.getResourceType(any[ResourceTypeName]) + } thenReturn IO.pure(Option(fakeWorkspaceResourceType)) + + when { + policyEvaluatorService.listUserResources(any[ResourceTypeName], any[WorkbenchUserId], any[SamRequestContext]) + } thenReturn IO.pure( + Vector( + UserResourcesResponse( + resourceId = ResourceId("cea587e9-9a8e-45b6-b985-9e3803754020"), + direct = RolesAndActions(Set.empty, Set.empty), + inherited = RolesAndActions(Set.empty, Set.empty), + public = RolesAndActions(Set.empty, Set.empty), + authDomainGroups = Set.empty, + missingAuthDomainGroups = Set.empty + ) + ) + ) + + MockSamDependencies( + mockResourceService, + policyEvaluatorService, + tosService, + userService, + statusService, + mockManagedGroupService, + directoryDAO, + policyDAO, + googleExt, + FakeOpenIDConnectConfiguration, + azureService + ) + } + + override def beforeAll(): Unit = { + startSam.unsafeToFuture() + startSam.start + sleep(5000) + } + + def startSam: IO[Http.ServerBinding] = + for { + binding <- IO + .fromFuture(IO(Http().newServerAt("localhost", 8080).bind(genSamRoutes(genSamDependencies, Generator.genWorkbenchUserBoth.sample.get).route))) + .onError { t: Throwable => + IO(logger.error("FATAL - failure starting http server", t)) *> IO.raiseError(t) + } + _ <- IO.fromFuture(IO(binding.whenTerminated)) + _ <- IO(system.terminate()) + } yield binding + + lazy val pactBrokerUrl: String = sys.env.getOrElse("PACT_BROKER_URL", "") + lazy val pactBrokerUser: String = sys.env.getOrElse("PACT_BROKER_USERNAME", "") + lazy val pactBrokerPass: String = sys.env.getOrElse("PACT_BROKER_PASSWORD", "") + lazy val branch: String = sys.env.getOrElse("BRANCH", "") + lazy val gitShaShort: String = sys.env.getOrElse("GIT_SHA_SHORT", "") + + override def provider: ProviderInfoBuilder = ProviderInfoBuilder( + name = "sam-provider", + // pactSource = PactSource.FileSource(Map("leo-consumer" -> new File("src/test/resources/leo-consumer-sam-provider.json"))) + pactSource = PactSource + .PactBrokerWithSelectors( + brokerUrl = pactBrokerUrl + ) + .withAuth(BasicAuth(pactBrokerUser, pactBrokerPass)) + ).withHost("localhost").withPort(8080) + + it should "Verify pacts" in { + verifyPacts( + providerBranch = if (branch.isEmpty) None else Some(Branch(branch)), + publishVerificationResults = Some( + PublishVerificationResults(gitShaShort, ProviderTags(branch)) + ), + providerVerificationOptions = Seq( + ProviderVerificationOption.SHOW_STACKTRACE, + // Exclude these Consumers from Pact Broker + ProviderVerificationOption.FILTER_CONSUMERS + .apply(Seq("Example App", "GoAdminService").toList) + ).toList, + verificationTimeout = Some(10.seconds) + ) + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 65a32ea3da..53428d4899 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -9,7 +9,6 @@ object Dependencies { val scalaCheckV = "1.14.3" val scalikejdbcVersion = "3.4.2" val postgresDriverVersion = "42.5.0" - val http4sVersion = "1.0.0-M32" val sentryVersion = "6.6.0" val workbenchUtilV = "0.6-74c9fc2" @@ -22,6 +21,7 @@ object Dependencies { val workbenchOpenTelemetryV = "0.3-0096bac" val monocleVersion = "2.0.5" val crlVersion = "1.2.4-SNAPSHOT" + val slf4jVersion = "2.0.6" val excludeAkkaActor = ExclusionRule(organization = "com.typesafe.akka", name = "akka-actor_2.12") val excludeAkkaProtobufV3 = ExclusionRule(organization = "com.typesafe.akka", name = "akka-protobuf-v3_2.12") @@ -117,6 +117,14 @@ object Dependencies { val opencensusStackDriverExporter: ModuleID = "io.opencensus" % "opencensus-exporter-trace-stackdriver" % "0.31.1" // excludeAll(excludIoGrpc, excludeCatsEffect) val opencensusLoggingExporter: ModuleID = "io.opencensus" % "opencensus-exporter-trace-logging" % "0.31.1" // excludeAll(excludIoGrpc, excludeCatsEffect) + val slf4jApi: ModuleID = "org.slf4j" % "slf4j-api" % slf4jVersion + val slf4jSimple: ModuleID = "org.slf4j" % "slf4j-simple" % slf4jVersion + + // pact deps + val pact4sV = "0.6.0" + val pact4sScalaTest = "io.github.jbwheatley" %% "pact4s-scalatest" % pact4sV % Test + val pact4sCirce = "io.github.jbwheatley" %% "pact4s-circe" % pact4sV + val circeCore = "io.circe" %% "circe-core" % "0.14.3" val openCensusDependencies = Seq( opencensusScalaCode, @@ -125,6 +133,14 @@ object Dependencies { opencensusLoggingExporter ) + val pact4sDependencies = Seq( + pact4sScalaTest, + pact4sCirce, + circeCore, + slf4jApi, + slf4jSimple + ) + val cloudResourceLib: ModuleID = "bio.terra" % "terra-cloud-resource-lib" % crlVersion excludeAll (excludeGoogleCloudResourceManager, excludeJerseyCore, excludeJerseyMedia, excludeSLF4J) val azureManagedApplications: ModuleID = diff --git a/project/Merging.scala b/project/Merging.scala index 6e3b29f3bf..88ec08ac17 100644 --- a/project/Merging.scala +++ b/project/Merging.scala @@ -4,9 +4,14 @@ object Merging { def customMergeStrategy(oldStrategy: (String) => MergeStrategy): (String => MergeStrategy) = { case PathList("org", "joda", "time", "base", "BaseDateTime.class") => MergeStrategy.first case PathList("io", "sundr", _ @_*) => MergeStrategy.first + case PathList("javax", "activation", _ @_*) => MergeStrategy.first + case PathList("javax", "xml", _ @_*) => MergeStrategy.first case PathList("google", "protobuf", _ @_*) => MergeStrategy.first + case x if x.endsWith("/ModuleUtil.class") => MergeStrategy.first case PathList("META-INF", "versions", "9", "module-info.class") => MergeStrategy.first case PathList("META-INF", "io.netty.versions.properties") => MergeStrategy.first + case PathList("META-INF", "kotlin-result.kotlin_module") => MergeStrategy.first + case PathList("mozilla", "public-suffix-list.txt") => MergeStrategy.first case "module-info.class" => MergeStrategy.discard case x => oldStrategy(x) diff --git a/project/Settings.scala b/project/Settings.scala index c8ce7a5e1e..3a90f3c976 100644 --- a/project/Settings.scala +++ b/project/Settings.scala @@ -2,10 +2,10 @@ import Dependencies._ import Merging._ import Testing._ import Version._ -import sbt.Keys.{scalacOptions, _} +import org.scalafmt.sbt.ScalafmtPlugin.autoImport.{scalafmtAll, scalafmtSbt} +import sbt.Keys._ import sbt.{Compile, Test, _} -import sbtassembly.AssemblyPlugin.autoImport.{assembly, _} -import org.scalafmt.sbt.ScalafmtPlugin.autoImport.{scalafmt, scalafmtAll, scalafmtCheck, scalafmtCheckAll, scalafmtOnCompile, scalafmtSbt, scalafmtSbtCheck} +import sbtassembly.AssemblyPlugin.autoImport._ object Settings { lazy val artifactory = "https://artifactory.broadinstitute.org/artifactory/" @@ -78,4 +78,14 @@ object Settings { name := "sam", libraryDependencies ++= rootDependencies ) ++ commonAssemblySettings ++ rootVersionSettings + + val pact4sSettings = commonSettings ++ List( + libraryDependencies ++= pact4sDependencies, + + /** Invoking pact tests from root project (sbt "project pact" test) will launch tests in a separate JVM context that ensures contracts are written to the + * pact/target/pacts folder. Otherwise, contracts will be written to the root folder. + */ + Test / fork := true, + publish / skip := true + ) ++ commonAssemblySettings ++ rootVersionSettings } diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockSamRoutes.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockSamRoutes.scala new file mode 100644 index 0000000000..6ce8954cd1 --- /dev/null +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockSamRoutes.scala @@ -0,0 +1,122 @@ +package org.broadinstitute.dsde.workbench.sam.api + +import akka.actor.ActorSystem +import akka.event.Logging.LogLevel +import akka.event.{Logging, LoggingAdapter} +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ +import akka.http.scaladsl.model._ +import akka.http.scaladsl.server +import akka.http.scaladsl.server.Directives._ +import akka.http.scaladsl.server.RouteResult.Complete +import akka.http.scaladsl.server.directives.{DebuggingDirectives, LogEntry, LoggingMagnet} +import akka.http.scaladsl.server.{Directive0, ExceptionHandler} +import akka.stream.Materializer +import akka.stream.scaladsl.Sink +import cats.effect.IO +import com.typesafe.scalalogging.LazyLogging +import org.broadinstitute.dsde.workbench.model.{ErrorReport, WorkbenchExceptionWithErrorReport} +import org.broadinstitute.dsde.workbench.oauth2.OpenIDConnectConfiguration +import org.broadinstitute.dsde.workbench.openTelemetry.OpenTelemetryMetrics +import org.broadinstitute.dsde.workbench.sam._ +import org.broadinstitute.dsde.workbench.sam.api.MockSamRoutes._ +import org.broadinstitute.dsde.workbench.sam.azure.{AzureRoutes, AzureService} +import org.broadinstitute.dsde.workbench.sam.config.{LiquibaseConfig, TermsOfServiceConfig} +import org.broadinstitute.dsde.workbench.sam.dataAccess.DirectoryDAO +import org.broadinstitute.dsde.workbench.sam.service._ + +import scala.concurrent.{ExecutionContext, Future} + +/** Created by dvoet on 5/17/17. + */ +abstract class MockSamRoutes( + val resourceService: ResourceService, + val userService: UserService, + val statusService: StatusService, + val managedGroupService: ManagedGroupService, + val termsOfServiceConfig: TermsOfServiceConfig, + val directoryDAO: DirectoryDAO, + val policyEvaluatorService: PolicyEvaluatorService, + val tosService: TosService, + val liquibaseConfig: LiquibaseConfig, + val oidcConfig: OpenIDConnectConfiguration, + val azureService: Option[AzureService] +)(implicit + val system: ActorSystem, + val materializer: Materializer, + val executionContext: ExecutionContext, + val openTelemetry: OpenTelemetryMetrics[IO] +) extends LazyLogging + with ResourceRoutes + with UserRoutes + with MockStatusRoutes + with TermsOfServiceRoutes + with ExtensionRoutes + with ManagedGroupRoutes + with AdminRoutes + with AzureRoutes { + + def route: server.Route = (logRequestResult & handleExceptions(myExceptionHandler)) { + oidcConfig.swaggerRoutes("swagger/api-docs.yaml") ~ + oidcConfig.oauth2Routes ~ + statusRoutes ~ + termsOfServiceRoutes ~ + withExecutionContext(ExecutionContext.global) { + withSamRequestContext { samRequestContext => + pathPrefix("register")(userRoutes(samRequestContext)) ~ + pathPrefix("api") { + // IMPORTANT - all routes under /api must have an active user + withActiveUser(samRequestContext) { samUser => + val samRequestContextWithUser = samRequestContext.copy(samUser = Option(samUser)) + resourceRoutes(samUser, samRequestContextWithUser) ~ + adminRoutes(samUser, samRequestContextWithUser) ~ + extensionRoutes(samUser, samRequestContextWithUser) ~ + groupRoutes(samUser, samRequestContextWithUser) ~ + apiUserRoutes(samUser, samRequestContextWithUser) ~ + azureRoutes(samUser, samRequestContextWithUser) + } + } + } + } + } + + // basis for logRequestResult lifted from http://stackoverflow.com/questions/32475471/how-does-one-log-akka-http-client-requests + private def logRequestResult: Directive0 = { + def entityAsString(entity: HttpEntity): Future[String] = + entity.dataBytes + .map(_.decodeString(entity.contentType.charsetOption.getOrElse(HttpCharsets.`UTF-8`).value)) + .runWith(Sink.head) + + def myLoggingFunction(logger: LoggingAdapter)(req: HttpRequest)(res: Any): Unit = { + val entry = res match { + case Complete(resp) => + val logLevel: LogLevel = resp.status.intValue / 100 match { + case 5 => Logging.ErrorLevel + case 4 => Logging.InfoLevel + case _ => Logging.DebugLevel + } + entityAsString(resp.entity).map(data => LogEntry(s"${req.method} ${req.uri}: ${resp.status} entity: $data", logLevel)) + case other => + Future.successful(LogEntry(s"$other", Logging.DebugLevel)) // I don't really know when this case happens + } + entry.map(_.logTo(logger)) + } + + DebuggingDirectives.logRequestResult(LoggingMagnet(log => myLoggingFunction(log))) + } + + def statusCodeCreated[T](response: T): (StatusCode, T) = (StatusCodes.Created, response) + +} + +object MockSamRoutes { + protected[sam] val myExceptionHandler = { + import org.broadinstitute.dsde.workbench.model.ErrorReportJsonSupport._ + + ExceptionHandler { + case withErrorReport: WorkbenchExceptionWithErrorReport => + complete((withErrorReport.errorReport.statusCode.getOrElse(StatusCodes.InternalServerError), withErrorReport.errorReport)) + case e: Throwable => + complete((StatusCodes.InternalServerError, ErrorReport(e))) + } + } +} diff --git a/src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockStatusRoutes.scala b/src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockStatusRoutes.scala new file mode 100644 index 0000000000..9a066cd3ef --- /dev/null +++ b/src/test/scala/org/broadinstitute/dsde/workbench/sam/api/MockStatusRoutes.scala @@ -0,0 +1,42 @@ +package org.broadinstitute.dsde.workbench.sam.api + +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ +import akka.http.scaladsl.model.StatusCodes +import akka.http.scaladsl.server +import akka.http.scaladsl.server.Directives._ +import org.broadinstitute.dsde.workbench.sam.service.StatusService +import org.broadinstitute.dsde.workbench.util.health.StatusJsonSupport._ + +import scala.concurrent.ExecutionContext + +trait MockStatusRoutes { + val statusService: StatusService + implicit val executionContext: ExecutionContext + + def statusRoutes: server.Route = + pathPrefix("status") { + pathEndOrSingleSlash { + get { + complete { + statusService.getStatus().map { statusResponse => + val httpStatus = if (statusResponse.ok) { + StatusCodes.OK + } else { + StatusCodes.InternalServerError + } + (httpStatus, statusResponse) + } + } + } + } + } ~ + pathPrefix("version") { + pathEndOrSingleSlash { + get { + complete { + (StatusCodes.OK, BuildTimeVersion.versionJson) + } + } + } + } +}